diff --git a/.github/workflows/draft-release-notes-workflow.yml b/.github/workflows/draft-release-notes-workflow.yml index 891dcb9cb0..57f00a8fbb 100644 --- a/.github/workflows/draft-release-notes-workflow.yml +++ b/.github/workflows/draft-release-notes-workflow.yml @@ -16,6 +16,6 @@ jobs: with: config-name: draft-release-notes-config.yml tag: (None) - version: 1.3.0.0 + version: 2.0.0.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/sql-odbc-release-workflow.yml b/.github/workflows/sql-odbc-release-workflow.yml index b929507c4b..98f3ab4bb1 100644 --- a/.github/workflows/sql-odbc-release-workflow.yml +++ b/.github/workflows/sql-odbc-release-workflow.yml @@ -12,7 +12,7 @@ env: ODBC_BUILD_PATH: "./build/odbc/build" AWS_SDK_INSTALL_PATH: "./build/aws-sdk/install" PLUGIN_NAME: opensearch-sql-odbc - OD_VERSION: 1.3.0.0 + OD_VERSION: 2.0.0.0 jobs: build-mac: diff --git a/.github/workflows/sql-odbc-rename-and-release-workflow.yml b/.github/workflows/sql-odbc-rename-and-release-workflow.yml index e97bf5012c..72b2cbe362 100644 --- a/.github/workflows/sql-odbc-rename-and-release-workflow.yml +++ b/.github/workflows/sql-odbc-rename-and-release-workflow.yml @@ -8,7 +8,7 @@ on: - rename* env: - OD_VERSION: 1.3.0.0 + OD_VERSION: 2.0.0.0 jobs: upload-odbc: diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 7097475aa6..985ce5a28b 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -3,14 +3,13 @@ name: SQL Java CI on: [push, pull_request] env: - OPENSEARCH_VERSION: '1.3.0-SNAPSHOT' + OPENSEARCH_VERSION: '2.0.0-SNAPSHOT' jobs: build: strategy: matrix: java: - - 8 - 11 - 14 runs-on: ubuntu-latest diff --git a/.github/workflows/sql-workbench-release-workflow.yml b/.github/workflows/sql-workbench-release-workflow.yml index 3a9353acbe..287f1210d9 100644 --- a/.github/workflows/sql-workbench-release-workflow.yml +++ b/.github/workflows/sql-workbench-release-workflow.yml @@ -7,8 +7,8 @@ on: env: PLUGIN_NAME: query-workbench-dashboards - OPENSEARCH_VERSION: '1.x' - OPENSEARCH_PLUGIN_VERSION: 1.3.0.0 + OPENSEARCH_VERSION: 'main' + OPENSEARCH_PLUGIN_VERSION: 2.0.0.0 jobs: @@ -38,7 +38,7 @@ jobs: - name: Setup Node uses: actions/setup-node@v1 with: - node-version: '10.24.1' + node-version: '14.18.2' - name: Move Workbench to Plugins Dir run: | diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index a534dac831..e824900e50 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -4,8 +4,8 @@ on: [pull_request, push] env: PLUGIN_NAME: query-workbench-dashboards - OPENSEARCH_VERSION: '1.x' - OPENSEARCH_PLUGIN_VERSION: 1.3.0.0 + OPENSEARCH_VERSION: 'main' + OPENSEARCH_PLUGIN_VERSION: 2.0.0.0 jobs: @@ -27,7 +27,7 @@ jobs: - name: Setup Node uses: actions/setup-node@v1 with: - node-version: '10.24.1' + node-version: '14.18.2' - name: Move Workbench to Plugins Dir run: | diff --git a/.gitignore b/.gitignore index 7eacb4168b..19689a12b0 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ gen */.venv */__pycache__ *.iml +.DS_Store diff --git a/bi-connectors/TableauConnector/opensearch_sql_jdbc/manifest.xml b/bi-connectors/TableauConnector/opensearch_sql_jdbc/manifest.xml index 427e9237a6..1ea29b9e82 100644 --- a/bi-connectors/TableauConnector/opensearch_sql_jdbc/manifest.xml +++ b/bi-connectors/TableauConnector/opensearch_sql_jdbc/manifest.xml @@ -1,6 +1,6 @@ - + diff --git a/build.gradle b/build.gradle index cb140922eb..5c3222e8ce 100644 --- a/build.gradle +++ b/build.gradle @@ -6,7 +6,7 @@ buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "1.3.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.0.0-SNAPSHOT") } repositories { diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle new file mode 100644 index 0000000000..c882e03029 --- /dev/null +++ b/buildSrc/build.gradle @@ -0,0 +1,12 @@ +apply plugin: 'groovy' +apply plugin: 'java' + + +repositories { + mavenCentral() +} + +dependencies { + implementation gradleApi() + implementation localGroovy() +} \ No newline at end of file diff --git a/buildSrc/src/main/groovy/com/wiredforcode/spawn/DefaultSpawnTask.groovy b/buildSrc/src/main/groovy/com/wiredforcode/spawn/DefaultSpawnTask.groovy new file mode 100644 index 0000000000..6d71a6410a --- /dev/null +++ b/buildSrc/src/main/groovy/com/wiredforcode/spawn/DefaultSpawnTask.groovy @@ -0,0 +1,18 @@ +package com.wiredforcode.gradle.spawn + +import org.gradle.api.DefaultTask +import org.gradle.api.tasks.Internal + + +class DefaultSpawnTask extends DefaultTask { + + @Internal + String pidLockFileName = '.pid.lock' + @Internal + String directory = '.' + + @Internal + File getPidFile() { + return new File(directory, pidLockFileName) + } +} diff --git a/buildSrc/src/main/groovy/com/wiredforcode/spawn/KillProcessTask.groovy b/buildSrc/src/main/groovy/com/wiredforcode/spawn/KillProcessTask.groovy new file mode 100644 index 0000000000..15c60b1aec --- /dev/null +++ b/buildSrc/src/main/groovy/com/wiredforcode/spawn/KillProcessTask.groovy @@ -0,0 +1,23 @@ +package com.wiredforcode.gradle.spawn + +import org.gradle.api.tasks.TaskAction + +class KillProcessTask extends DefaultSpawnTask { + @TaskAction + void kill() { + def pidFile = getPidFile() + if(!pidFile.exists()) { + logger.quiet "No server running!" + return + } + + def pid = pidFile.text + def process = "kill $pid".execute() + + try { + process.waitFor() + } finally { + pidFile.delete() + } + } +} diff --git a/buildSrc/src/main/groovy/com/wiredforcode/spawn/SpawnPlugin.groovy b/buildSrc/src/main/groovy/com/wiredforcode/spawn/SpawnPlugin.groovy new file mode 100644 index 0000000000..bc67617c52 --- /dev/null +++ b/buildSrc/src/main/groovy/com/wiredforcode/spawn/SpawnPlugin.groovy @@ -0,0 +1,18 @@ +package com.wiredforcode.gradle.spawn + +import org.gradle.api.Plugin +import org.gradle.api.Project + + +class SpawnPlugin implements Plugin { + @Override + void apply(Project project) { + project.with { + ext.SpawnProcessTask = SpawnProcessTask + ext.KillProcessTask = KillProcessTask + + task('spawnProcess', type: SpawnProcessTask) + task('killProcess', type: KillProcessTask) + } + } +} diff --git a/buildSrc/src/main/groovy/com/wiredforcode/spawn/SpawnProcessTask.groovy b/buildSrc/src/main/groovy/com/wiredforcode/spawn/SpawnProcessTask.groovy new file mode 100644 index 0000000000..1b3d0c4f00 --- /dev/null +++ b/buildSrc/src/main/groovy/com/wiredforcode/spawn/SpawnProcessTask.groovy @@ -0,0 +1,103 @@ +package com.wiredforcode.gradle.spawn + +import org.gradle.api.GradleException +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.TaskAction + +class SpawnProcessTask extends DefaultSpawnTask { + @Input + String command + @Input + String ready + @Internal + List outputActions = new ArrayList() + + @Input + Map environmentVariables = new HashMap() + + void environmentVariable(String key, Object value) { + environmentVariables.put(key, String.valueOf(value)) + } + + SpawnProcessTask() { + description = "Spawn a new server process in the background." + } + + void withOutput(Closure outputClosure) { + outputActions.add(outputClosure) + } + + @TaskAction + void spawn() { + if (!(command && ready)) { + throw new GradleException("Ensure that mandatory fields command and ready are set.") + } + + def pidFile = getPidFile() + if (pidFile.exists()) throw new GradleException("Server already running!") + + def process = buildProcess(directory, command) + waitToProcessReadyOrClosed(process) + } + + private void waitToProcessReadyOrClosed(Process process) { + boolean isReady = waitUntilIsReadyOrEnd(process) + if (isReady) { + stampLockFile(pidFile, process) + } else { + checkForAbnormalExit(process) + } + } + + private void checkForAbnormalExit(Process process) { + try { + process.waitFor() + def exitValue = process.exitValue() + if (exitValue) { + throw new GradleException("The process terminated unexpectedly - status code ${exitValue}") + } + } catch (IllegalThreadStateException ignored) { + } + } + + private boolean waitUntilIsReadyOrEnd(Process process) { + def line + def reader = new BufferedReader(new InputStreamReader(process.getInputStream())) + boolean isReady = false + while (!isReady && (line = reader.readLine()) != null) { + logger.quiet line + runOutputActions(line) + if (line.contains(ready)) { + logger.quiet "$command is ready." + isReady = true + } + } + isReady + } + + def runOutputActions(String line) { + outputActions.each { Closure outputAction -> + outputAction.call(line) + } + } + + private Process buildProcess(String directory, String command) { + def builder = new ProcessBuilder(command.split(' ')) + builder.redirectErrorStream(true) + builder.environment().putAll(environmentVariables) + builder.directory(new File(directory)) + builder.start() + } + + private File stampLockFile(File pidFile, Process process) { + pidFile << extractPidFromProcess(process) + } + + private int extractPidFromProcess(Process process) { + def pidField = process.class.getDeclaredField('pid') + pidField.accessible = true + + return pidField.getInt(process) + } +} diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/com.wiredforcode.spawn.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/com.wiredforcode.spawn.properties new file mode 100644 index 0000000000..ff84035f8e --- /dev/null +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/com.wiredforcode.spawn.properties @@ -0,0 +1 @@ +implementation-class=com.wiredforcode.gradle.spawn.SpawnPlugin \ No newline at end of file diff --git a/common/build.gradle b/common/build.gradle index 3f23b5cd31..f7143a30ac 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -23,7 +23,7 @@ */ plugins { - id 'java' + id 'java-library' id "io.freefair.lombok" } @@ -32,9 +32,9 @@ repositories { } dependencies { - compile "org.antlr:antlr4-runtime:4.7.1" - compile group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' - compile group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.17.1' + api "org.antlr:antlr4-runtime:4.7.1" + api group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' + api group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.17.1' - testCompile group: 'junit', name: 'junit', version: '4.13.2' + testImplementation group: 'junit', name: 'junit', version: '4.13.2' } diff --git a/core/build.gradle b/core/build.gradle index 32a14bad3b..6c16b5f1f4 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -23,7 +23,7 @@ */ plugins { - id 'java' + id 'java-library' id "io.freefair.lombok" id 'jacoco' } @@ -39,19 +39,19 @@ repositories { //} dependencies { - compile group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' - compile group: 'org.springframework', name: 'spring-context', version: '5.2.19.RELEASE' - compile group: 'org.springframework', name: 'spring-beans', version: '5.2.19.RELEASE' - compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.10' - compile group: 'com.facebook.presto', name: 'presto-matching', version: '0.240' - compile group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1' - compile project(':common') + api group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' + api group: 'org.springframework', name: 'spring-context', version: '5.2.19.RELEASE' + api group: 'org.springframework', name: 'spring-beans', version: '5.2.19.RELEASE' + api group: 'org.apache.commons', name: 'commons-lang3', version: '3.10' + api group: 'com.facebook.presto', name: 'presto-matching', version: '0.240' + api group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1' + api project(':common') testImplementation('org.junit.jupiter:junit-jupiter:5.6.2') - testCompile group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' - testCompile group: 'org.springframework', name: 'spring-test', version: '5.2.19.RELEASE' - testCompile group: 'org.mockito', name: 'mockito-core', version: '3.3.3' - testCompile group: 'org.mockito', name: 'mockito-junit-jupiter', version: '3.3.3' + testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' + testImplementation group: 'org.springframework', name: 'spring-test', version: '5.2.19.RELEASE' + testImplementation group: 'org.mockito', name: 'mockito-core', version: '3.3.3' + testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: '3.3.3' } test { diff --git a/doctest/build.gradle b/doctest/build.gradle index f310806147..b417920df4 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -7,8 +7,8 @@ import java.util.concurrent.Callable import org.opensearch.gradle.testclusters.RunTask plugins { - id "com.wiredforcode.spawn" version "0.8.2" id 'base' + id 'com.wiredforcode.spawn' } apply plugin: 'opensearch.testclusters' diff --git a/doctest/resources/ml-commons/opensearch-ml-1.3.0.0-SNAPSHOT.zip b/doctest/resources/ml-commons/opensearch-ml-2.0.0.0-SNAPSHOT.zip similarity index 96% rename from doctest/resources/ml-commons/opensearch-ml-1.3.0.0-SNAPSHOT.zip rename to doctest/resources/ml-commons/opensearch-ml-2.0.0.0-SNAPSHOT.zip index 2d4ee43458..e0e722c1c7 100644 Binary files a/doctest/resources/ml-commons/opensearch-ml-1.3.0.0-SNAPSHOT.zip and b/doctest/resources/ml-commons/opensearch-ml-2.0.0.0-SNAPSHOT.zip differ diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 5c2d1cf016..e708b1c023 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index cc6edfec26..52006bf626 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,8 +1,7 @@ # Copyright OpenSearch Contributors # SPDX-License-Identifier: Apache-2.0 - -distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-all.zip distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3-bin.zip zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 2cafa621a9..025c045cb5 100755 --- a/gradlew +++ b/gradlew @@ -1,5 +1,4 @@ #!/usr/bin/env sh - # # Copyright OpenSearch Contributors # SPDX-License-Identifier: Apache-2.0 @@ -10,8 +9,6 @@ # # Modifications Copyright OpenSearch Contributors. See # GitHub history for details. -# - # # Copyright 2015 the original author or authors. # @@ -94,6 +91,7 @@ esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + # Determine the Java command to use to start the JVM. if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then @@ -137,10 +135,11 @@ if $darwin; then GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" fi -# For Cygwin, switch paths to Windows format before running java -if $cygwin ; then +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then APP_HOME=`cygpath --path --mixed "$APP_HOME"` CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` # We build the pattern for arguments to be converted via cygpath @@ -166,19 +165,19 @@ if $cygwin ; then else eval `echo args$i`="\"$arg\"" fi - i=$((i+1)) + i=`expr $i + 1` done case $i in - (0) set -- ;; - (1) set -- "$args0" ;; - (2) set -- "$args0" "$args1" ;; - (3) set -- "$args0" "$args1" "$args2" ;; - (4) set -- "$args0" "$args1" "$args2" "$args3" ;; - (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; esac fi @@ -187,14 +186,9 @@ save () { for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done echo " " } -APP_ARGS=$(save "$@") +APP_ARGS=`save "$@"` # Collect all arguments for the java command, following the shell quoting and substitution rules eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" -# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong -if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then - cd "$(dirname "$0")" -fi - exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat index e55187938f..fc8457a4e9 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -1,4 +1,4 @@ -@rem +rem @rem Copyright OpenSearch Contributors @rem SPDX-License-Identifier: Apache-2.0 @rem @@ -8,8 +8,6 @@ @rem @rem Modifications Copyright OpenSearch Contributors. See @rem GitHub history for details. -@rem - @rem @rem Copyright 2015 the original author or authors. @rem @@ -41,6 +39,9 @@ if "%DIRNAME%" == "" set DIRNAME=. set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" @@ -49,7 +50,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init +if "%ERRORLEVEL%" == "0" goto execute echo. echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. @@ -63,7 +64,7 @@ goto fail set JAVA_HOME=%JAVA_HOME:"=% set JAVA_EXE=%JAVA_HOME%/bin/java.exe -if exist "%JAVA_EXE%" goto init +if exist "%JAVA_EXE%" goto execute echo. echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% @@ -73,28 +74,14 @@ echo location of your Java installation. goto fail -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - :execute @rem Setup the command line set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + @rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* :end @rem End local scope for the variables with windows NT shell diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 6079eede9f..26b404ddd2 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -57,30 +57,31 @@ configurations.all { } dependencies { - testCompile group: 'org.opensearch.test', name: 'framework', version: "${opensearch_version}" - testCompile group: 'org.opensearch.client', name: 'opensearch-rest-high-level-client', version: "${opensearch_version}" - testCompile group: 'org.opensearch.client', name: 'opensearch-rest-client', version: "${opensearch_version}" - testCompile group: 'org.hamcrest', name: 'hamcrest', version: '2.1' - testCompile group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.17.1' - testCompile project(':plugin') - testCompile project(':legacy') + testImplementation group: 'org.opensearch.test', name: 'framework', version: "${opensearch_version}" + testImplementation group: 'org.opensearch.client', name: 'opensearch-rest-high-level-client', version: "${opensearch_version}" + testImplementation group: 'org.opensearch.client', name: 'opensearch-rest-client', version: "${opensearch_version}" + testImplementation group: 'org.hamcrest', name: 'hamcrest', version: '2.1' + implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.17.1' + testImplementation project(':plugin') + testImplementation project(':legacy') testImplementation('org.junit.jupiter:junit-jupiter-api:5.6.2') testRuntimeOnly('org.junit.jupiter:junit-jupiter-engine:5.6.2') // JDBC drivers for comparison test. Somehow Apache Derby throws security permission exception. - testCompile fileTree('../sql-jdbc/build/libs') { + testImplementation fileTree('../sql-jdbc/build/libs') { include '*.jar' builtBy 'compileJdbc' } - testCompile group: 'com.h2database', name: 'h2', version: '2.1.210' - testCompile group: 'org.xerial', name: 'sqlite-jdbc', version: '3.28.0' - testCompile group: 'com.google.code.gson', name: 'gson', version: '2.8.9' + testImplementation group: 'com.h2database', name: 'h2', version: '2.1.210' + testImplementation group: 'org.xerial', name: 'sqlite-jdbc', version: '3.28.0' + testImplementation group: 'com.google.code.gson', name: 'gson', version: '2.8.9' } dependencyLicenses.enabled = false testingConventions.enabled = false checkstyleTest.ignoreFailures = true forbiddenApisTest.enabled = false +thirdPartyAudit.enabled = false compileTestJava { options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) @@ -143,27 +144,6 @@ integTest { } -task docTest(type: RestIntegTestTask) { - dependsOn ':plugin:bundlePlugin' - - systemProperty 'tests.security.manager', 'false' - systemProperty('project.root', project.projectDir.absolutePath) - - // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for - // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time. - doFirst { systemProperty 'cluster.debug', getDebug() } - - if (System.getProperty("test.debug") != null) { - jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' - } - - include 'org/opensearch/sql/doctest/**/*IT.class' - exclude 'org/opensearch/sql/correctness/**/*IT.class' - exclude 'org/opensearch/sql/ppl/**/*IT.class' - exclude 'org/opensearch/sql/sql/**/*IT.class' - exclude 'org/opensearch/sql/legacy/**/*IT.class' -} - task comparisonTest(type: RestIntegTestTask) { dependsOn ':plugin:bundlePlugin' diff --git a/integ-test/src/test/java/org/opensearch/sql/correctness/CorrectnessIT.java b/integ-test/src/test/java/org/opensearch/sql/correctness/CorrectnessIT.java index 1fe2170a70..a6f3e561b3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/correctness/CorrectnessIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/correctness/CorrectnessIT.java @@ -10,10 +10,7 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import com.google.common.collect.Maps; -import java.io.IOException; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.URL; + import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -27,23 +24,19 @@ import org.json.JSONObject; import org.junit.Test; import org.opensearch.client.RestClient; -import org.opensearch.common.Strings; -import org.opensearch.common.transport.TransportAddress; import org.opensearch.sql.correctness.report.TestReport; import org.opensearch.sql.correctness.runner.ComparisonTest; import org.opensearch.sql.correctness.runner.connection.DBConnection; import org.opensearch.sql.correctness.runner.connection.JDBCConnection; import org.opensearch.sql.correctness.runner.connection.OpenSearchConnection; import org.opensearch.sql.correctness.testset.TestDataSet; -import org.opensearch.sql.legacy.CustomExternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; -import org.opensearch.test.TestCluster; /** * Correctness integration test by performing comparison test with other databases. */ @OpenSearchIntegTestCase.SuiteScopeTestCase -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 3, supportsDedicatedMasters = false, transportClientRatio = 1) +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 3, supportsDedicatedMasters = false) @ThreadLeakScope(ThreadLeakScope.Scope.NONE) public class CorrectnessIT extends OpenSearchIntegTestCase { @@ -140,26 +133,4 @@ private String reportFileName() { String dateTime = df.format(new Date()); return "report_" + dateTime + ".json"; } - - @Override - protected TestCluster buildTestCluster(Scope scope, long seed) throws IOException { - - String clusterAddresses = System.getProperty(TESTS_CLUSTER); - - if (Strings.hasLength(clusterAddresses)) { - String[] stringAddresses = clusterAddresses.split(","); - TransportAddress[] transportAddresses = new TransportAddress[stringAddresses.length]; - int i = 0; - for (String stringAddress : stringAddresses) { - URL url = new URL("http://" + stringAddress); - InetAddress inetAddress = InetAddress.getByName(url.getHost()); - transportAddresses[i++] = - new TransportAddress(new InetSocketAddress(inetAddress, url.getPort())); - } - return new CustomExternalTestCluster(createTempDir(), externalClusterClientSettings(), - transportClientPlugins(), transportAddresses); - } - return super.buildTestCluster(scope, seed); - } - } diff --git a/integ-test/src/test/java/org/opensearch/sql/correctness/tests/ComparisonTestTest.java b/integ-test/src/test/java/org/opensearch/sql/correctness/tests/ComparisonTestTest.java index ec0c90f2f5..03c3967544 100644 --- a/integ-test/src/test/java/org/opensearch/sql/correctness/tests/ComparisonTestTest.java +++ b/integ-test/src/test/java/org/opensearch/sql/correctness/tests/ComparisonTestTest.java @@ -8,7 +8,7 @@ import static java.util.Arrays.asList; import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.anyString; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -17,7 +17,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; -import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.junit.MockitoJUnitRunner; import org.opensearch.sql.correctness.report.ErrorTestCase; import org.opensearch.sql.correctness.report.FailedTestCase; import org.opensearch.sql.correctness.report.SuccessTestCase; diff --git a/integ-test/src/test/java/org/opensearch/sql/correctness/tests/JDBCConnectionTest.java b/integ-test/src/test/java/org/opensearch/sql/correctness/tests/JDBCConnectionTest.java index 1c253c9d82..0e70066136 100644 --- a/integ-test/src/test/java/org/opensearch/sql/correctness/tests/JDBCConnectionTest.java +++ b/integ-test/src/test/java/org/opensearch/sql/correctness/tests/JDBCConnectionTest.java @@ -7,8 +7,8 @@ package org.opensearch.sql.correctness.tests; import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.anyInt; -import static org.mockito.Matchers.anyString; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -30,7 +30,7 @@ import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; import org.mockito.Mock; -import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.OngoingStubbing; import org.opensearch.sql.correctness.runner.connection.JDBCConnection; import org.opensearch.sql.correctness.runner.resultset.DBResult; diff --git a/integ-test/src/test/java/org/opensearch/sql/correctness/tests/OpenSearchConnectionTest.java b/integ-test/src/test/java/org/opensearch/sql/correctness/tests/OpenSearchConnectionTest.java index 10e533c310..73659525f2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/correctness/tests/OpenSearchConnectionTest.java +++ b/integ-test/src/test/java/org/opensearch/sql/correctness/tests/OpenSearchConnectionTest.java @@ -7,7 +7,7 @@ package org.opensearch.sql.correctness.tests; import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.any; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -24,7 +24,7 @@ import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; import org.mockito.Mock; -import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.junit.MockitoJUnitRunner; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.RestClient; diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/admin/MonitoringIT.java b/integ-test/src/test/java/org/opensearch/sql/doctest/admin/MonitoringIT.java deleted file mode 100644 index e391379244..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/admin/MonitoringIT.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.admin; - -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.CURL_REQUEST; -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; -import static org.opensearch.sql.legacy.metrics.MetricName.DEFAULT_CURSOR_REQUEST_COUNT_TOTAL; -import static org.opensearch.sql.legacy.metrics.MetricName.DEFAULT_CURSOR_REQUEST_TOTAL; -import static org.opensearch.sql.legacy.metrics.MetricName.FAILED_REQ_COUNT_CB; -import static org.opensearch.sql.legacy.metrics.MetricName.FAILED_REQ_COUNT_CUS; -import static org.opensearch.sql.legacy.metrics.MetricName.FAILED_REQ_COUNT_SYS; -import static org.opensearch.sql.legacy.metrics.MetricName.REQ_COUNT_TOTAL; -import static org.opensearch.sql.legacy.metrics.MetricName.REQ_TOTAL; -import static org.opensearch.sql.legacy.plugin.RestSqlStatsAction.STATS_API_ENDPOINT; - -import org.opensearch.sql.doctest.core.DocTest; -import org.opensearch.sql.doctest.core.annotation.DocTestConfig; -import org.opensearch.sql.doctest.core.annotation.Section; -import org.opensearch.sql.doctest.core.builder.Requests; -import org.opensearch.sql.doctest.core.request.SqlRequest; -import org.opensearch.sql.doctest.core.response.DataTable; -import org.opensearch.sql.legacy.metrics.MetricName; - -/** - * Doc test for plugin monitoring functionality - */ -@DocTestConfig(template = "admin/monitoring.rst") -public class MonitoringIT extends DocTest { - - @Section - public void nodeStats() { - section( - title("Node Stats"), - description( - "The meaning of fields in the response is as follows:\n\n" + fieldDescriptions()), - example( - description(), - getStats(), - queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ) - ); - } - - private String fieldDescriptions() { - DataTable table = new DataTable(new String[] {"Field name", "Description"}); - table.addRow(row(REQ_TOTAL, "Total count of request")); - table.addRow(row(REQ_COUNT_TOTAL, "Total count of request within the interval")); - table.addRow(row(DEFAULT_CURSOR_REQUEST_TOTAL, "Total count of simple cursor request")); - table.addRow(row(DEFAULT_CURSOR_REQUEST_COUNT_TOTAL, - "Total count of simple cursor request within the interval")); - table.addRow(row(FAILED_REQ_COUNT_SYS, - "Count of failed request due to system error within the interval")); - table.addRow(row(FAILED_REQ_COUNT_CUS, - "Count of failed request due to bad request within the interval")); - table.addRow( - row(FAILED_REQ_COUNT_CB, "Indicate if plugin is being circuit broken within the interval")); - - return table.toString(); - } - - private String[] row(MetricName name, String description) { - return new String[] {name.getName(), description}; - } - - private Requests getStats() { - return new Requests(restClient(), new SqlRequest("GET", STATS_API_ENDPOINT, "")); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/beyond/FullTextIT.java b/integ-test/src/test/java/org/opensearch/sql/doctest/beyond/FullTextIT.java deleted file mode 100644 index ded89f9ab2..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/beyond/FullTextIT.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.beyond; - -import org.opensearch.sql.doctest.core.DocTest; -import org.opensearch.sql.doctest.core.annotation.DocTestConfig; -import org.opensearch.sql.doctest.core.annotation.Section; - -@DocTestConfig(template = "beyond/fulltext.rst", testData = {"accounts.json"}) -public class FullTextIT extends DocTest { - - @Section(1) - public void matchQuery() { - section( - title("Match Query"), - description( - "Match query is the standard query for full-text search in OpenSearch. Both ``MATCHQUERY`` and", - "``MATCH_QUERY`` are functions for performing match query." - ), - example( - description( - "Both functions can accept field name as first argument and a text as second argument."), - post(multiLine( - "SELECT account_number, address", - "FROM accounts", - "WHERE MATCH_QUERY(address, 'Holmes')" - )) - ), - example( - description( - "Both functions can also accept single argument and be used in the following manner."), - post(multiLine( - "SELECT account_number, address", - "FROM accounts", - "WHERE address = MATCH_QUERY('Holmes')" - )) - ) - ); - } - - @Section(2) - public void multiMatchQuery() { - section( - title("Multi-match Query"), - description( - "Besides match query against a single field, you can search for a text with multiple fields.", - "Function ``MULTI_MATCH``, ``MULTIMATCH`` and ``MULTIMATCHQUERY`` are provided for this." - ), - example( - description( - "Each preceding function accepts ``query`` for a text and ``fields`` for field names or pattern", - "that the text given is searched against. For example, the following query is searching for", - "documents in index accounts with 'Dale' as either firstname or lastname." - ), - post(multiLine( - "SELECT firstname, lastname", - "FROM accounts", - "WHERE MULTI_MATCH('query'='Dale', 'fields'='*name')" - )) - ) - ); - } - - @Section(3) - public void queryStringQuery() { - section( - title("Query String Query"), - description( - "Query string query parses and splits a query string provided based on Lucene query string syntax.", - "The mini language supports logical connectives, wildcard, regex and proximity search. Please refer", - "to official documentation for more details. Note that an error is thrown in the case of any invalid", - "syntax in query string." - ), - example( - description( - "``QUERY`` function accepts query string and returns true or false respectively for document", - "that matches the query string or not." - ), - post(multiLine( - "SELECT account_number, address", - "FROM accounts", - "WHERE QUERY('address:Lane OR address:Street')" - )) - ) - ); - } - - @Section(4) - public void matchPhraseQuery() { - section( - title("Match Phrase Query"), - description( - "Match phrase query is similar to match query but it is used for matching exact phrases.", - "``MATCHPHRASE``, ``MATCH_PHRASE`` and ``MATCHPHRASEQUERY`` are provided for this purpose." - ), - example( - description(), - post(multiLine( - "SELECT account_number, address", - "FROM accounts", - "WHERE MATCH_PHRASE(address, '880 Holmes Lane')" - )) - ) - ); - } - - @Section(5) - public void scoreQuery() { - section( - title("Score Query"), - description( - "OpenSearch supports to wrap a filter query so as to return a relevance score along with", - "every matching document. ``SCORE``, ``SCOREQUERY`` and ``SCORE_QUERY`` can be used for this." - ), - example( - description( - "The first argument is a match query expression and the second argument is for an optional", - "floating point number to boost the score. The default value is 1.0. Apart from this, an", - "implicit variable ``_score`` is available so you can return score for each document or", - "use it for sorting." - ), - post(multiLine( - "SELECT account_number, address, _score", - "FROM accounts", - "WHERE SCORE(MATCH_QUERY(address, 'Lane'), 0.5) OR", - " SCORE(MATCH_QUERY(address, 'Street'), 100)", - "ORDER BY _score" - )) - ) - ); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/beyond/PartiQLIT.java b/integ-test/src/test/java/org/opensearch/sql/doctest/beyond/PartiQLIT.java deleted file mode 100644 index dfce25e0ec..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/beyond/PartiQLIT.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.beyond; - -import static org.opensearch.sql.doctest.core.TestData.TEST_DATA_FOLDER_ROOT; -import static org.opensearch.sql.util.TestUtils.getResourceFilePath; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import org.opensearch.sql.doctest.core.DocTest; -import org.opensearch.sql.doctest.core.annotation.DocTestConfig; -import org.opensearch.sql.doctest.core.annotation.Section; -import org.opensearch.sql.doctest.core.builder.Example; -import org.opensearch.sql.legacy.utils.JsonPrettyFormatter; - -@DocTestConfig(template = "beyond/partiql.rst", testData = {"employees_nested.json"}) -public class PartiQLIT extends DocTest { - - @Section(1) - public void showTestData() { - section( - title("Test Data"), - description( - "The test index ``employees_nested`` used by all examples in this document is very similar to", - "the one used in official PartiQL documentation." - ), - createDummyExampleForTestData("employees_nested.json") - ); - } - - @Section(2) - public void queryNestedCollection() { - section( - title("Querying Nested Collection"), - description( - "In SQL-92, a database table can only have tuples that consists of scalar values.", - "PartiQL extends SQL-92 to allow you query and unnest nested collection conveniently.", - "In OpenSearch world, this is very useful for index with object or nested field." - ), - example( - title("Unnesting a Nested Collection"), - description( - "In the following example, it finds nested document (project) with field value (name)", - "that satisfies the predicate (contains 'security'). Note that because each parent document", - "can have more than one nested documents, the matched nested document is flattened. In other", - "word, the final result is the Cartesian Product between parent and nested documents." - ), - post(multiLine( - "SELECT e.name AS employeeName,", - " p.name AS projectName", - "FROM employees_nested AS e,", - " e.projects AS p", - "WHERE p.name LIKE '%security%'" - )) - ), - /* - Issue: https://github.com/opendistro-for-elasticsearch/sql/issues/397 - example( - title("Preserving Parent Information with LEFT JOIN"), - description( - "The query in the preceding example is very similar to traditional join queries, except ``ON`` clause missing.", - "This is because it is implicitly in the nesting of nested documents (projects) into parent (employee). Therefore,", - "you can use ``LEFT JOIN`` to preserve the information in parent document associated." - ), - post( - "SELECT e.id AS id, " + - " e.name AS employeeName, " + - " e.title AS title, " + - " p.name AS projectName " + - "FROM employees_nested AS e " + - "LEFT JOIN e.projects AS p" - ) - )*/ - example( - title("Unnesting in Existential Subquery"), - description( - "Alternatively, a nested collection can be unnested in subquery to check if it", - "satisfies a condition." - ), - post(multiLine( - "SELECT e.name AS employeeName", - "FROM employees_nested AS e", - "WHERE EXISTS (", - " SELECT *", - " FROM e.projects AS p", - " WHERE p.name LIKE '%security%'", - ")" - )) - )/*, - Issue: https://github.com/opendistro-for-elasticsearch/sql/issues/398 - example( - title("Aggregating over a Nested Collection"), - description( - "After unnested, a nested collection can be aggregated just like a regular field." - ), - post(multiLine( - "SELECT", - " e.name AS employeeName,", - " COUNT(p) AS cnt", - "FROM employees_nested AS e,", - " e.projects AS p", - "WHERE p.name LIKE '%security%'", - "GROUP BY e.id, e.name", - "HAVING COUNT(p) >= 1" - ) - )) - */ - ); - } - - private Example createDummyExampleForTestData(String fileName) { - Example example = new Example(); - example.setTitle("Employees"); - example.setDescription(""); - example.setResult(parseJsonFromTestData(fileName)); - return example; - } - - /** - * Concat and pretty format document at odd number line in bulk request file - */ - private String parseJsonFromTestData(String fileName) { - Path path = Paths.get(getResourceFilePath(TEST_DATA_FOLDER_ROOT + fileName)); - try { - List lines = Files.readAllLines(path); - String json = IntStream.range(0, lines.size()). - filter(i -> i % 2 == 1). - mapToObj(lines::get). - collect(Collectors.joining(",", "{\"employees\":[", "]}")); - return JsonPrettyFormatter.format(json); - } catch (IOException e) { - throw new IllegalStateException("Failed to load test data: " + path, e); - } - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/DocTest.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/DocTest.java deleted file mode 100644 index afce22c0f3..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/DocTest.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core; - -import static com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope; -import static java.nio.file.StandardOpenOption.APPEND; -import static org.opensearch.test.OpenSearchIntegTestCase.Scope.SUITE; - -import com.carrotsearch.randomizedtesting.AnnotatedMethodProvider; -import com.carrotsearch.randomizedtesting.TestMethodAndParams; -import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering; -import com.carrotsearch.randomizedtesting.annotations.TestMethodProviders; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import java.io.IOException; -import java.io.PrintWriter; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Comparator; -import org.opensearch.client.RestClient; -import org.opensearch.common.Strings; -import org.opensearch.common.transport.TransportAddress; -import org.opensearch.sql.doctest.core.annotation.DocTestConfig; -import org.opensearch.sql.doctest.core.annotation.Section; -import org.opensearch.sql.doctest.core.builder.DocBuilder; -import org.opensearch.sql.doctest.core.markup.Document; -import org.opensearch.sql.doctest.core.markup.RstDocument; -import org.opensearch.sql.legacy.CustomExternalTestCluster; -import org.opensearch.sql.legacy.TestUtils; -import org.opensearch.test.OpenSearchIntegTestCase; -import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; -import org.opensearch.test.TestCluster; - -/** - * Documentation test base class - */ -@TestMethodProviders({DocTest.SectionMethod.class}) -@TestCaseOrdering(DocTest.SectionOrder.class) -@OpenSearchIntegTestCase.SuiteScopeTestCase -@ClusterScope(scope = SUITE, numDataNodes = 1, supportsDedicatedMasters = false, transportClientRatio = 1) -@ThreadLeakScope(Scope.NONE) -public abstract class DocTest extends OpenSearchIntegTestCase implements DocBuilder { - - @Override - protected void setupSuiteScopeCluster() { - DocTestConfig config = getClass().getAnnotation(DocTestConfig.class); - loadTestData(config); - copyTemplateToDocument(config); - } - - @Override - public RestClient restClient() { - return getRestClient(); - } - - @Override - public Document openDocument() { - DocTestConfig config = getClass().getAnnotation(DocTestConfig.class); - Path docPath = absolutePath(config.template()); - try { - PrintWriter docWriter = new PrintWriter(Files.newBufferedWriter(docPath, APPEND)); - return new RstDocument(docWriter); - } catch (IOException e) { - throw new IllegalStateException("Failed to open document file " + docPath, e); - } - } - - private void loadTestData(DocTestConfig config) { - String[] testFilePaths = config.testData(); - TestData testData = new TestData(testFilePaths); - testData.loadToES(this); - } - - private void copyTemplateToDocument(DocTestConfig config) { - Path docPath = absolutePath(config.template()); - Template template = new Template(config.template()); - template.copyToDocument(docPath); - } - - /** - * Method annotated by {@link Section} will be treated as test method. - */ - public static class SectionMethod extends AnnotatedMethodProvider { - public SectionMethod() { - super(Section.class); - } - } - - /** - * Test methods will execute in order defined by value in {@link Section} annotation. - */ - public static class SectionOrder implements Comparator { - @Override - public int compare(TestMethodAndParams method1, TestMethodAndParams method2) { - return Integer.compare(order(method1), order(method2)); - } - - private int order(TestMethodAndParams method) { - Section section = method.getTestMethod().getAnnotation(Section.class); - return section.value(); - } - } - - private Path absolutePath(String templateRelativePath) { - return Paths.get(TestUtils.getResourceFilePath(DOCUMENT_FOLDER_ROOT + templateRelativePath)); - } - - @Override - protected TestCluster buildTestCluster(Scope scope, long seed) throws IOException { - - String clusterAddresses = System.getProperty(TESTS_CLUSTER); - - if (Strings.hasLength(clusterAddresses)) { - String[] stringAddresses = clusterAddresses.split(","); - TransportAddress[] transportAddresses = new TransportAddress[stringAddresses.length]; - int i = 0; - for (String stringAddress : stringAddresses) { - URL url = new URL("http://" + stringAddress); - InetAddress inetAddress = InetAddress.getByName(url.getHost()); - transportAddresses[i++] = - new TransportAddress(new InetSocketAddress(inetAddress, url.getPort())); - } - return new CustomExternalTestCluster(createTempDir(), externalClusterClientSettings(), - transportClientPlugins(), transportAddresses); - } - return super.buildTestCluster(scope, seed); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/Template.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/Template.java deleted file mode 100644 index c84a2c1a11..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/Template.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core; - -import static java.nio.file.StandardCopyOption.COPY_ATTRIBUTES; -import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import org.opensearch.sql.legacy.utils.StringUtils; -import org.opensearch.sql.util.TestUtils; - -/** - * Abstraction for document template file - */ -public class Template { - - private static final String TEMPLATE_FOLDER_ROOT = "src/test/resources/doctest/templates/"; - - private final Path templateFullPath; - - public Template(String templateRelativePath) { - this.templateFullPath = - Paths.get(TestUtils.getResourceFilePath(TEMPLATE_FOLDER_ROOT + templateRelativePath)); - } - - /** - * Copy template file to target document. Replace it if existing. - * - * @param docFullPath full path of target document - */ - public void copyToDocument(Path docFullPath) { - try { - Files.createDirectories(docFullPath.getParent()); - Files.copy(templateFullPath, docFullPath, REPLACE_EXISTING, COPY_ATTRIBUTES); - } catch (IOException e) { - throw new IllegalStateException(StringUtils.format( - "Failed to copy from template [%s] to document file [%s]", templateFullPath, docFullPath), - e); - } - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/TestData.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/TestData.java deleted file mode 100644 index 962ce1ecff..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/TestData.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core; - -import static org.opensearch.sql.util.TestUtils.createIndexByRestClient; -import static org.opensearch.sql.util.TestUtils.getResourceFilePath; -import static org.opensearch.sql.util.TestUtils.loadDataByRestClient; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import org.opensearch.sql.legacy.utils.StringUtils; - -/** - * Test data for document generation - */ -public class TestData { - - public static final String MAPPINGS_FOLDER_ROOT = "src/test/resources/doctest/mappings/"; - public static final String TEST_DATA_FOLDER_ROOT = "src/test/resources/doctest/testdata/"; - - private final String[] testFilePaths; - - public TestData(String[] testFilePaths) { - this.testFilePaths = testFilePaths; - } - - /** - * Load test data in file to Elaticsearch cluster via client. - * - * @param test current test instance - */ - public void loadToES(DocTest test) { - for (String filePath : testFilePaths) { - String indexName = indexName(filePath); - try { - createIndexByRestClient(test.restClient(), indexName, getIndexMapping(filePath)); - loadDataByRestClient(test.restClient(), indexName, TEST_DATA_FOLDER_ROOT + filePath); - } catch (Exception e) { - throw new IllegalStateException(StringUtils.format( - "Failed to load mapping and test filePath from %s", filePath), e); - } - test.ensureGreen(indexName); - } - } - - /** - * Use file name (without file extension) as index name implicitly. - * For example, for 'testdata/accounts.json', 'accounts' will be used. - */ - private String indexName(String filePath) { - return filePath.substring( - filePath.lastIndexOf(File.separatorChar) + 1, - filePath.lastIndexOf('.') - ); - } - - private String getIndexMapping(String filePath) throws IOException { - Path path = Paths.get(getResourceFilePath(MAPPINGS_FOLDER_ROOT + filePath)); - if (Files.notExists(path)) { - return ""; - } - return new String(Files.readAllBytes(path)); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/annotation/DocTestConfig.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/annotation/DocTestConfig.java deleted file mode 100644 index 0a2ddf8a2d..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/annotation/DocTestConfig.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.annotation; - -import static java.lang.annotation.ElementType.TYPE; -import static java.lang.annotation.RetentionPolicy.RUNTIME; - -import java.lang.annotation.Retention; -import java.lang.annotation.Target; - -/** - * Configuration to initializing the set up for a doc test. - */ -@Retention(RUNTIME) -@Target(value = TYPE) -public @interface DocTestConfig { - - /** - * Path of the template - * - * @return path - */ - String template(); - - /** - * Path of the test data used. - * - * @return path - */ - String[] testData() default {}; - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/annotation/Section.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/annotation/Section.java deleted file mode 100644 index 215542c36e..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/annotation/Section.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.annotation; - -import static java.lang.annotation.ElementType.METHOD; -import static java.lang.annotation.RetentionPolicy.RUNTIME; - -import java.lang.annotation.Retention; -import java.lang.annotation.Target; - -/** - * This annotation is used to indicate current method is a valid method for doc generation - * and it is supposed to run in the specified order. - */ -@Retention(RUNTIME) -@Target(value = METHOD) -public @interface Section { - - /** - * @return section order - */ - int value() default 0; - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Body.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Body.java deleted file mode 100644 index 45da0618df..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Body.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.builder; - -import java.util.Arrays; -import java.util.stream.Collectors; - -/** - * Request body. - */ -class Body { - - private final String[] fieldValues; - - /** - * Request body built from field value pairs. - * - * @param fieldValues field and values in "'field": 'value'" format that can assemble to JSON directly - */ - Body(String... fieldValues) { - this.fieldValues = fieldValues; - } - - @Override - public String toString() { - return Arrays.stream(fieldValues).collect(Collectors.joining(",", "{", "}")); - } -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/DocBuilder.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/DocBuilder.java deleted file mode 100644 index 1c6361fa65..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/DocBuilder.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.builder; - -import static org.opensearch.sql.doctest.core.request.SqlRequest.UrlParam; -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.OPENSEARCH_DASHBOARD_REQUEST; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.TABLE_RESPONSE; -import static org.opensearch.sql.legacy.plugin.RestSqlAction.EXPLAIN_API_ENDPOINT; -import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; -import static org.opensearch.sql.plugin.rest.RestQuerySettingsAction.LEGACY_SQL_SETTINGS_API_ENDPOINT; - - -import com.google.common.base.Strings; -import java.util.Arrays; -import org.opensearch.client.RestClient; -import org.opensearch.sql.doctest.core.markup.Document; -import org.opensearch.sql.doctest.core.request.SqlRequest; -import org.opensearch.sql.doctest.core.request.SqlRequestFormat; -import org.opensearch.sql.doctest.core.response.SqlResponseFormat; -import org.opensearch.sql.legacy.utils.StringUtils; - -/** - * Build document by custom DSL. To make it more readable, each doc test needs to implement this interface - * and provide things required, such as client connection and document handle. As benefit, they can use the - * DSL to build document in readable and fluent way. - */ -public interface DocBuilder { - - String DOCUMENT_FOLDER_ROOT = "/docs/user/"; - String IMAGE_FOLDER_PATH = DOCUMENT_FOLDER_ROOT + "img/"; - - /** - * Get client connection to cluster for sending request - * - * @return REST client - */ - RestClient restClient(); - - /** - * Open document file to write - * - * @return document class - */ - Document openDocument(); - - default void section(String title, String description, Example... examples) { - section(title, description, new String[0], examples); - } - - /** - * Entry point to start building document by DSL. - * Each section consists of: - * 1. Title - * 2. Description - * 3. Example(s) - * 3.1 Description - * 3.2 [Sample request] - * 3.3 [Explain request] - * 3.4 [Explain output] - * 3.5 [Result set] - * - * @param title title of the section - * @param description description paragraph - * @param examples examples for the section - */ - default void section(String title, String description, String[] images, Example... examples) { - try (Document document = openDocument()) { - document.section(title); - - if (!description.isEmpty()) { - document.subSection("Description").paragraph(description); - } - - if (images.length > 0) { - document.subSection("Syntax"); - for (String image : images) { - // Convert image name ex. "rdd/queryStatement.png" to "queryStatement" as description. - String imageDesc = image.substring(image.lastIndexOf('/') + 1, image.lastIndexOf('.')); - document.image("Rule ``" + imageDesc + "``", IMAGE_FOLDER_PATH + image); - } - } - - for (int i = 0; i < examples.length; i++) { - String exampleTitle; - if (examples.length > 1) { - exampleTitle = "Example " + (i + 1); - } else { - exampleTitle = "Example"; - } - if (!Strings.isNullOrEmpty(examples[i].getTitle())) { - exampleTitle += (": " + examples[i].getTitle()); - } - document.subSection(exampleTitle); - - Example example = examples[i]; - if (!example.getDescription().isEmpty()) { - document.paragraph(example.getDescription()); - } - - document.codeBlock("SQL query", example.getQuery()). - codeBlock("Explain query", example.getExplainQuery()). - codeBlock("Explain", example.getExplainResult()); - - if (example.isTable()) { - document.table("Result set", example.getResult()); - } else { - document.codeBlock("Result set", example.getResult()); - } - } - } - } - - default Example example(String description, Requests requests) { - return example("", description, requests); - } - - /** - * Construct an example by default query and explain format - */ - default Example example(String title, String description, Requests requests) { - return example(title, description, requests, - queryFormat(OPENSEARCH_DASHBOARD_REQUEST, TABLE_RESPONSE), - explainFormat(IGNORE_REQUEST, PRETTY_JSON_RESPONSE) - ); - } - - default Example example(String description, - Requests requests, - Formats queryFormat, - Formats explainFormat) { - return example("", description, requests, queryFormat, explainFormat); - } - - default Example example(String title, - String description, - Requests requests, - Formats queryFormat, - Formats explainFormat) { - Example example = new Example(); - example.setTitle(title); - example.setDescription(description); - example.setQuery(queryFormat.format(requests.query())); - example.setTable(queryFormat.isTableFormat()); - example.setResult(queryFormat.format(requests.queryResponse())); - example.setExplainQuery(explainFormat.format(requests.explain())); - example.setExplainResult(explainFormat.format(requests.explainResponse())); - return example; - } - - /** - * Simple method just for readability - */ - default String title(String title) { - return title; - } - - default String description(String... sentences) { - return String.join(" ", sentences); - } - - default String[] images(String... images) { - return images; - } - - default Formats queryFormat(SqlRequestFormat requestFormat, SqlResponseFormat responseFormat) { - return new Formats(requestFormat, responseFormat); - } - - default Formats explainFormat(SqlRequestFormat requestFormat, SqlResponseFormat responseFormat) { - return new Formats(requestFormat, responseFormat); - } - - default Requests get(String sql) { - Body body = new Body("\"query\":\"" + sql + "\""); - return new Requests( - restClient(), - new SqlRequest("GET", QUERY_API_ENDPOINT, "", new UrlParam("sql", sql)), - new SqlRequest("POST", EXPLAIN_API_ENDPOINT, body.toString()) - ); - } - - default Requests put(String name, Object value) { - String setting = value == null ? - StringUtils.format("\"%s\": {\"%s\": null}", "transient", name) : - StringUtils.format("\"%s\": {\"%s\": \"%s\"}", "transient", name, value); - return new Requests( - restClient(), - new SqlRequest("PUT", LEGACY_SQL_SETTINGS_API_ENDPOINT, new Body(setting).toString()), - null - ); - } - - default String multiLine(String... lines) { - return String.join("\\n", lines); - } - - /** - * Query by a simple SQL is too common and deserve a dedicated overload method - */ - default Requests post(String sql, UrlParam... params) { - return post(new Body("\"query\":\"" + sql + "\""), params); - } - - default Requests post(Body body, UrlParam... params) { - String bodyStr = body.toString(); - return new Requests( - restClient(), - new SqlRequest("POST", QUERY_API_ENDPOINT, bodyStr, params), - new SqlRequest("POST", EXPLAIN_API_ENDPOINT, bodyStr) - ); - } - - default Body body(String... fieldValues) { - return new Body(fieldValues); - } - - default UrlParam[] params(String... keyValues) { - return Arrays.stream(keyValues).map(UrlParam::new).toArray(UrlParam[]::new); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Example.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Example.java deleted file mode 100644 index d35165845c..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Example.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.builder; - -/** - * Example value object. - */ -public class Example { - - /** - * Title for the example section - */ - private String title; - - /** - * Description for the example - */ - private String description; - - /** - * Sample SQL query - */ - private String query; - - /** - * Query result set - */ - private String result; - - /** - * Is result set formatted in table (markup handle table in different way - */ - private boolean isTable; - - /** - * Explain query correspondent to the sample query - */ - private String explainQuery; - - /** - * Result of explain - */ - private String explainResult; - - public String getTitle() { - return title; - } - - public String getDescription() { - return description; - } - - public String getQuery() { - return query; - } - - public String getResult() { - return result; - } - - public boolean isTable() { - return isTable; - } - - public String getExplainQuery() { - return explainQuery; - } - - public String getExplainResult() { - return explainResult; - } - - public void setTitle(String title) { - this.title = title; - } - - public void setDescription(String description) { - this.description = description; - } - - public void setQuery(String query) { - this.query = query; - } - - public void setResult(String result) { - this.result = result; - } - - public void setTable(boolean table) { - isTable = table; - } - - public void setExplainQuery(String explainQuery) { - this.explainQuery = explainQuery; - } - - public void setExplainResult(String explainResult) { - this.explainResult = explainResult; - } -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Formats.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Formats.java deleted file mode 100644 index c281dd2c50..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Formats.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.builder; - -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.TABLE_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.TABLE_UNSORTED_RESPONSE; - -import org.opensearch.sql.doctest.core.request.SqlRequest; -import org.opensearch.sql.doctest.core.request.SqlRequestFormat; -import org.opensearch.sql.doctest.core.response.SqlResponse; -import org.opensearch.sql.doctest.core.response.SqlResponseFormat; - -/** - * Request and response format tuple. - */ -class Formats { - - private final SqlRequestFormat requestFormat; - private final SqlResponseFormat responseFormat; - - Formats(SqlRequestFormat requestFormat, SqlResponseFormat responseFormat) { - this.requestFormat = requestFormat; - this.responseFormat = responseFormat; - } - - String format(SqlRequest request) { - return requestFormat.format(request); - } - - String format(SqlResponse response) { - return responseFormat.format(response); - } - - boolean isTableFormat() { - return responseFormat == TABLE_RESPONSE || responseFormat == TABLE_UNSORTED_RESPONSE; - } -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/ListItems.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/ListItems.java deleted file mode 100644 index a3e0418956..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/ListItems.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.builder; - -import org.opensearch.sql.legacy.utils.StringUtils; - -/** - * Item list - */ -public class ListItems { - private final StringBuilder list = new StringBuilder(); - private int index = 0; - - public void addItem(String text) { - list.append(index()).append(text).append('\n'); - } - - private String index() { - index++; - return StringUtils.format("%d. ", index); - } - - @Override - public String toString() { - return list.toString(); - } -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Requests.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Requests.java deleted file mode 100644 index cba0ee8428..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/builder/Requests.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.builder; - -import java.util.Objects; -import org.opensearch.client.RestClient; -import org.opensearch.sql.doctest.core.request.SqlRequest; -import org.opensearch.sql.doctest.core.response.SqlResponse; - -/** - * Query and explain request tuple. - */ -public class Requests { - - private final RestClient client; - private final SqlRequest query; - private final SqlRequest explain; - - public Requests(RestClient client, SqlRequest query) { - this(client, query, SqlRequest.NONE); - } - - public Requests(RestClient client, SqlRequest query, SqlRequest explain) { - this.client = client; - this.query = query; - this.explain = explain; - } - - public SqlRequest query() { - return query; - } - - public SqlResponse queryResponse() { - Objects.requireNonNull(query, "Query request is required"); - return query.send(client); - } - - public SqlRequest explain() { - return explain; - } - - public SqlResponse explainResponse() { - if (explain == SqlRequest.NONE) { - return SqlResponse.NONE; - } - return explain.send(client); - } -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/markup/Document.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/markup/Document.java deleted file mode 100644 index eeb5f12f44..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/markup/Document.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.markup; - -import java.io.Closeable; - -/** - * Document for different format and markup - */ -public interface Document extends Closeable { - - /** - * Remove checked IOException in method signature. - */ - @Override - void close(); - - Document section(String title); - - Document subSection(String title); - - Document paragraph(String text); - - Document codeBlock(String description, String code); - - Document table(String description, String table); - - Document image(String description, String filePath); - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/markup/RstDocument.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/markup/RstDocument.java deleted file mode 100644 index 96930851f8..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/markup/RstDocument.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.markup; - -import com.google.common.base.Strings; -import java.io.PrintWriter; - -/** - * ReStructure Text document - */ -public class RstDocument implements Document { - - private final PrintWriter docWriter; - - public RstDocument(PrintWriter docWriter) { - this.docWriter = docWriter; - } - - @Override - public Document section(String title) { - return printTitleWithUnderline(title, "="); - } - - @Override - public Document subSection(String title) { - return printTitleWithUnderline(title, "-"); - } - - @Override - public Document paragraph(String text) { - return println(text); - } - - @Override - public Document codeBlock(String description, String code) { - if (!Strings.isNullOrEmpty(code)) { - return println(description + "::", indent(code)); - } - return this; - } - - @Override - public Document table(String description, String table) { - if (!Strings.isNullOrEmpty(table)) { - // RST table is different and not supposed to indent - return println(description + ":", table); - } - return this; - } - - @Override - public Document image(String description, String filePath) { - return println( - description + ":", - ".. image:: " + filePath - ); - } - - @Override - public void close() { - docWriter.close(); - } - - private Document printTitleWithUnderline(String title, String underlineChar) { - return print( - title, - Strings.repeat(underlineChar, title.length()) - ); - } - - /** - * Print each line with a blank line at last - */ - private Document print(String... lines) { - for (String line : lines) { - docWriter.println(line); - } - docWriter.println(); - return this; - } - - /** - * Print each line with a blank line followed - */ - private Document println(String... lines) { - for (String line : lines) { - docWriter.println(line); - docWriter.println(); - } - return this; - } - - private String indent(String text) { - return "\t" + text.replaceAll("\\n", "\n\t"); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/request/SqlRequest.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/request/SqlRequest.java deleted file mode 100644 index 6459f72d0c..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/request/SqlRequest.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.request; - -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.OPENSEARCH_DASHBOARD_REQUEST; - -import java.io.IOException; -import org.opensearch.client.Request; -import org.opensearch.client.RequestOptions; -import org.opensearch.client.ResponseException; -import org.opensearch.client.RestClient; -import org.opensearch.sql.doctest.core.response.SqlResponse; -import org.opensearch.sql.legacy.utils.StringUtils; - -/** - * Request to SQL plugin to isolate OpenSearch native request - */ -public class SqlRequest { - - public static final SqlRequest NONE = null; - - /** - * Native OpenSearch request object - */ - private final Request request; - - public SqlRequest(String method, String endpoint, String body, UrlParam... params) { - this.request = makeRequest(method, endpoint, body, params); - } - - /** - * Send request to OpenSearch via client and create response for it. - * - * @param client restful client connection - * @return sql response - */ - public SqlResponse send(RestClient client) { - try { - return new SqlResponse(client.performRequest(request)); - } catch (IOException e) { - // Some test may expect failure - if (e instanceof ResponseException) { - return new SqlResponse(((ResponseException) e).getResponse()); - } - - throw new IllegalStateException(StringUtils.format( - "Exception occurred during sending request %s", OPENSEARCH_DASHBOARD_REQUEST.format(this)), e); - } - } - - /** - * Expose request for request formatter. - * - * @return native OpenSearch format - */ - public Request request() { - return request; - } - - private Request makeRequest(String method, String endpoint, String body, UrlParam[] params) { - Request request = new Request(method, endpoint); - request.setJsonEntity(body); - for (UrlParam param : params) { - request.addParameter(param.key, param.value); - } - - RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); - restOptionsBuilder.addHeader("Content-Type", "application/json"); - request.setOptions(restOptionsBuilder); - return request; - } - - public static class UrlParam { - private String key; - private String value; - - public UrlParam(String key, String value) { - this.key = key; - this.value = value; - } - - public UrlParam(String keyValue) { - int equality = keyValue.indexOf('='); - if (equality == -1) { - throw new IllegalArgumentException(String.format( - "Key value pair is in bad format [%s]", keyValue)); - } - - this.key = keyValue.substring(0, equality); - this.value = keyValue.substring(equality + 1); - } - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/request/SqlRequestFormat.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/request/SqlRequestFormat.java deleted file mode 100644 index e4c152a49e..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/request/SqlRequestFormat.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.request; - -import static java.util.stream.Collectors.joining; - -import com.google.common.base.Charsets; -import com.google.common.io.CharStreams; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import org.apache.http.Header; -import org.json.JSONObject; -import org.opensearch.client.Request; -import org.opensearch.sql.legacy.utils.JsonPrettyFormatter; -import org.opensearch.sql.legacy.utils.StringUtils; - -/** - * Different SQL request formats. - */ -public enum SqlRequestFormat { - - IGNORE_REQUEST { - @Override - public String format(SqlRequest request) { - return ""; - } - }, - CURL_REQUEST { - @Override - public String format(SqlRequest sqlRequest) { - Request request = sqlRequest.request(); - StringBuilder str = new StringBuilder(); - str.append(">> curl "); - - List
headers = request.getOptions().getHeaders(); - if (!headers.isEmpty()) { - str.append(headers.stream(). - map(header -> StringUtils.format("-H '%s: %s'", header.getName(), header.getValue())). - collect(joining(" ", "", " "))); - } - - str.append(StringUtils.format("-X %s ", request.getMethod())). - append("localhost:9200").append(request.getEndpoint()); - - if (!request.getParameters().isEmpty()) { - str.append(formatParams(request.getParameters())); - } - - String body = body(request); - if (!body.isEmpty()) { - str.append(" -d '"). - append(body). - append('\''); - } - return str.toString(); - } - }, - OPENSEARCH_DASHBOARD_REQUEST { - @Override - public String format(SqlRequest sqlRequest) { - Request request = sqlRequest.request(); - StringBuilder str = new StringBuilder(); - str.append(request.getMethod()). - append(" "). - append(request.getEndpoint()); - - if (!request.getParameters().isEmpty()) { - str.append(formatParams(request.getParameters())); - } - - str.append('\n'). - append(body(request)); - return str.toString(); - } - }; - - /** - * Format SQL request to specific format for documentation. - * - * @param request sql request - * @return string in specific format - */ - public abstract String format(SqlRequest request); - - @SuppressWarnings("UnstableApiUsage") - protected String body(Request request) { - String body = ""; - try { - InputStream content = request.getEntity().getContent(); - String rawBody = CharStreams.toString(new InputStreamReader(content, Charsets.UTF_8)); - if (!rawBody.isEmpty()) { - JSONObject json = new JSONObject(rawBody); - String sql = json.optString("query"); // '\\n' in literal is replaced by '\n' after unquote - body = JsonPrettyFormatter.format(rawBody); - - // Format and replace multi-line sql literal - if (!sql.isEmpty() && sql.contains("\n")) { - String multiLineSql = - Arrays.stream(sql.split("\\n")). // '\\n' is to escape backslash in regex - collect(joining("\n\t", - "\"\"\"\n\t", - "\n\t\"\"\"")); - body = body.replace("\"" + sql.replace("\n", "\\n") + "\"", multiLineSql); - } - } - } catch (IOException e) { - throw new IllegalStateException("Failed to parse and format body from request", e); - } - return body; - } - - protected String formatParams(Map params) { - return params.entrySet().stream(). - map(e -> e.getKey() + "=" + e.getValue()). - collect(joining("&", "?", "")); - } -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/response/DataTable.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/response/DataTable.java deleted file mode 100644 index 1955f8fa94..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/response/DataTable.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.response; - -import com.google.common.base.Strings; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import org.opensearch.sql.legacy.utils.StringUtils; - -/** - * Data table that represent rows of data with a header. - * For now the format is actually in ReST and may need to decouple later. - */ -public class DataTable { - - private final int[] maxWidths; - private final Object[] header; - private final List rows; - - public DataTable(Object[] header) { - this.maxWidths = new int[header.length]; - this.header = header; - this.rows = new ArrayList<>(); - updateMaxWidthForEachColumn(header); - } - - public void addRow(Object[] row) { - rows.add(row); - updateMaxWidthForEachColumn(row); - } - - @Override - public String toString() { - StringBuilder str = new StringBuilder(); - String format = format(); - String separateLine1 = separateLine("-"); - String separateLine2 = separateLine("="); - - str.append(separateLine1). - append('\n'). - append(StringUtils.format(format, header)). - append('\n'). - append(separateLine2). - append('\n'); - - for (Object[] row : rows) { - str.append(StringUtils.format(format, row)). - append('\n'). - append(separateLine("-")). - append('\n'); - } - return str.toString(); - } - - private void updateMaxWidthForEachColumn(Object[] row) { - for (int i = 0; i < row.length; i++) { - maxWidths[i] = Math.max(maxWidths[i], String.valueOf(row[i]).length()); - } - } - - private String separateLine(String separator) { - return Arrays.stream(maxWidths). - mapToObj(width -> Strings.repeat(separator, width)). - collect(Collectors.joining("+", "+", "+")); - } - - /** - * Format as Java String.format needs to make use of auto pad feature. - * For example, to ensure width of 10 and pad spaces, we need to String.format("%10s", str); - */ - private String format() { - return Arrays.stream(maxWidths). - mapToObj(width -> "%" + width + "s"). - collect(Collectors.joining("|", "|", "|")); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/response/SqlResponse.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/response/SqlResponse.java deleted file mode 100644 index cddf3c9dc7..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/response/SqlResponse.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.response; - -import java.io.IOException; -import org.json.JSONException; -import org.json.JSONObject; -import org.opensearch.client.Response; -import org.opensearch.sql.util.TestUtils; - -/** - * Response from SQL plugin - */ -public class SqlResponse { - - public static final SqlResponse NONE = null; - - /** - * Native OpenSearch response - */ - private final Response response; - - public SqlResponse(Response response) { - this.response = response; - } - - /** - * Parse body in the response. - * - * @return response body - */ - public String body() { - try { - return replaceChangingFields(TestUtils.getResponseBody(response, true)); - } catch (IOException e) { - throw new IllegalStateException("Failed to read response body", e); - } - } - - /** - * In OpenSearch response, there is field changed between each query, such as "took". - * We have to replace those variants with fake constant to avoid re-generate documents. - * The order of fields in JSON is a little different from original because of internal - * key set in org.json. - */ - private String replaceChangingFields(String response) { - try { - JSONObject root = new JSONObject(response); - if (root.has("took")) { - root.put("took", 100); - } else { - return response; // return original response to minimize impact - } - - if (root.has("_shards")) { - JSONObject shards = root.getJSONObject("_shards"); - shards.put("total", 5); - shards.put("successful", 5); - } - return root.toString(); - } catch (JSONException e) { - // Response is not a valid JSON which is not our interest. - return response; - } - } -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/response/SqlResponseFormat.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/response/SqlResponseFormat.java deleted file mode 100644 index 3a054e59df..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/response/SqlResponseFormat.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.response; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import org.json.JSONArray; -import org.json.JSONObject; -import org.opensearch.sql.legacy.utils.JsonPrettyFormatter; -import org.opensearch.sql.legacy.utils.StringUtils; - -/** - * Different SQL response formats - */ -public enum SqlResponseFormat { - - IGNORE_RESPONSE { - @Override - public String format(SqlResponse sqlResponse) { - return ""; - } - }, - ORIGINAL_RESPONSE { - @Override - public String format(SqlResponse sqlResponse) { - return sqlResponse.body(); - } - }, - PRETTY_JSON_RESPONSE { - @Override - public String format(SqlResponse sqlResponse) { - String body = sqlResponse.body(); - try { - return JsonPrettyFormatter.format(body); - } catch (IOException e) { - throw new IllegalStateException( - StringUtils.format("Failed to pretty format response: %s", body), e); - } - } - }, - TABLE_RESPONSE { - @Override - public String format(SqlResponse sqlResponse) { - return format(sqlResponse, true); - } - }, - TABLE_UNSORTED_RESPONSE { - @Override - public String format(SqlResponse sqlResponse) { - return format(sqlResponse, false); - } - }; - - /** - * Format SQL response to specific format for documentation - * - * @param sqlResponse sql response - * @return string in specific format - */ - public abstract String format(SqlResponse sqlResponse); - - /** - * Note that we put this format() here because it's shared by two format enums. - * - * @param sqlResponse original response from plugin - * @param isSorted true to sort the result or just leave it as is - */ - protected String format(SqlResponse sqlResponse, boolean isSorted) { - JSONObject body = new JSONObject(sqlResponse.body()); - if (body.isNull("schema")) { - throw new IllegalStateException( - "Only JDBC response can be formatted to table: " + sqlResponse.body()); - } - - Object[] header = parseHeader(body.getJSONArray("schema")); - List rows = parseDataRows(body.getJSONArray("datarows"), isSorted); - - DataTable table = new DataTable(header); - for (Object[] row : rows) { - table.addRow(row); - } - return table.toString(); - } - - private Object[] parseHeader(JSONArray schema) { - Object[] header = new Object[schema.length()]; - for (int i = 0; i < header.length; i++) { - JSONObject nameType = schema.getJSONObject(i); - header[i] = nameType.optString("alias", nameType.getString("name")); - } - return header; - } - - private List parseDataRows(JSONArray rows, boolean isSorted) { - List rowsToSort = new ArrayList<>(); - for (Object row : rows) { - rowsToSort.add(((JSONArray) row).toList().toArray()); - } - - if (isSorted) { - sort(rowsToSort); - } - return rowsToSort; - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static void sort(List lists) { - lists.sort((list1, list2) -> { - if (list1 == null || list2 == null) { - return compareNullable(list1, list2); - } - - // Assume 2 lists are of same length and all elements are comparable - for (int i = 0; i < list1.length; i++) { - Comparable obj1 = (Comparable) list1[i]; - Comparable obj2 = (Comparable) list2[i]; - - if (obj1 == null || obj2 == null) { - return compareNullable(obj1, obj2); - } - - int result = obj1.compareTo(obj2); - if (result != 0) { - return result; - } - } - return 0; - }); - } - - /** - * Put NULL first (as smaller element) - */ - private static int compareNullable(Object obj1, Object obj2) { - if (obj1 == null && obj2 == null) { - return 0; - } else if (obj1 == null) { - return -1; - } else { // obj2 == null - return 1; - } - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/DataTableTest.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/DataTableTest.java deleted file mode 100644 index 8ffc4a6a6c..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/DataTableTest.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.test; - -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; - -import org.junit.Test; -import org.opensearch.sql.doctest.core.response.DataTable; - -/** - * Test cases for {@link DataTable} - */ -public class DataTableTest { - - @Test - public void testSingleColumnTable() { - DataTable table = new DataTable(new Object[] {"Test Table"}); - table.addRow(new Object[] {"this is a very long line"}); - - assertThat( - table.toString(), - is( - "+------------------------+\n" + - "| Test Table|\n" + - "+========================+\n" + - "|this is a very long line|\n" + - "+------------------------+\n" - ) - ); - } - - @Test - public void testTwoColumnsTable() { - DataTable table = new DataTable(new Object[] {"Test Table", "Very Long Title"}); - table.addRow(new Object[] {"this is a very long line", "short"}); - - assertThat( - table.toString(), - is( - "+------------------------+---------------+\n" + - "| Test Table|Very Long Title|\n" + - "+========================+===============+\n" + - "|this is a very long line| short|\n" + - "+------------------------+---------------+\n" - ) - ); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/DocBuilderTest.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/DocBuilderTest.java deleted file mode 100644 index d89066362b..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/DocBuilderTest.java +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.test; - -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.inOrder; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import org.apache.http.HttpEntity; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.InOrder; -import org.mockito.Mock; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.runners.MockitoJUnitRunner; -import org.mockito.stubbing.Answer; -import org.opensearch.client.Response; -import org.opensearch.client.RestClient; -import org.opensearch.sql.doctest.core.builder.DocBuilder; -import org.opensearch.sql.doctest.core.markup.Document; - -/** - * Test cases for {@link DocBuilder} - */ -@RunWith(MockitoJUnitRunner.class) -public class DocBuilderTest implements DocBuilder { - - private final String queryResponse = - "{\"schema\":[{\"name\":\"firstname\",\"type\":\"text\"}]," + - "\"datarows\":[[\"John\"]],\"total\":10,\"size\":1,\"status\":200}"; - - private final String explainResponse = - "{\"from\":0,\"size\":1,\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]}}"; - - @Mock - private Document document; - - private Verifier verifier; - - @Mock - private RestClient client; - - @Before - public void setUp() throws IOException { - when(document.section(any())).thenReturn(document); - when(document.subSection(any())).thenReturn(document); - when(document.paragraph(any())).thenReturn(document); - when(document.codeBlock(any(), any())).thenReturn(document); - when(document.table(any(), any())).thenReturn(document); - when(document.image(any(), any())).thenReturn(document); - verifier = new Verifier(document); - - when(client.performRequest(any())).then(new Answer() { - private int callCount = 0; - - @Override - public Response answer(InvocationOnMock invocationOnMock) throws IOException { - Response response = mock(Response.class); - HttpEntity entity = mock(HttpEntity.class); - when(response.getEntity()).thenReturn(entity); - - String body = (callCount++ == 0) ? queryResponse : explainResponse; - when(entity.getContent()).thenReturn(new ByteArrayInputStream(body.getBytes())); - return response; - } - }); - } - - @Test - public void sectionShouldIncludeTitleAndDescription() { - section( - title("Test"), - description("This is a test") - ); - - verifier.section("Test"). - subSection("Description"). - paragraph("This is a test"); - } - - @Test - public void sectionShouldIncludeMultiLineSql() { - section( - title("Test"), - description("This is a test"), - example( - description("This is an example for the test"), - post(multiLine( - "SELECT firstname", - "FROM accounts", - "WHERE age > 30") - ) - ) - ); - - verifier.section("Test"). - subSection("Description"). - paragraph("This is a test"). - subSection("Example"). - paragraph("This is an example for the test"). - codeBlock( - "SQL query", - "POST /_plugins/_sql\n" + - "{\n" + - " \"query\" : \"\"\"\n" + - "\tSELECT firstname\n" + - "\tFROM accounts\n" + - "\tWHERE age > 30\n" + - "\t\"\"\"\n" + - "}" - ); - } - - @Test - public void sectionShouldIncludeExample() { - section( - title("Test"), - description("This is a test"), - images("rdd/querySyntax.png"), - example( - description("This is an example for the test"), - post("SELECT firstname FROM accounts") - ) - ); - - verifier.section("Test"). - subSection("Description"). - paragraph("This is a test"). - image("Rule ``querySyntax``", "/docs/user/img/rdd/querySyntax.png"). - subSection("Example"). - paragraph("This is an example for the test"). - codeBlock( - "SQL query", - "POST /_plugins/_sql\n" + - "{\n" + - " \"query\" : \"SELECT firstname FROM accounts\"\n" + - "}" - ). - codeBlock( - "Explain", - "{\n" + - " \"from\" : 0,\n" + - " \"size\" : 1,\n" + - " \"_source\" : {\n" + - " \"includes\" : [\n" + - " \"firstname\"\n" + - " ],\n" + - " \"excludes\" : [ ]\n" + - " }\n" + - "}" - ).table( - "Result set", - "+---------+\n" + - "|firstname|\n" + - "+=========+\n" + - "| John|\n" + - "+---------+\n" - ); - } - - @Override - public RestClient restClient() { - return client; - } - - @Override - public Document openDocument() { - return document; - } - - private static class Verifier implements Document { - private final Document mock; - private final InOrder verifier; - - Verifier(Document mock) { - this.mock = mock; - this.verifier = inOrder(mock); - } - - @Override - public void close() { - verifier.verify(mock).close(); - } - - @Override - public Document section(String title) { - verifier.verify(mock).section(title); - return this; - } - - @Override - public Document subSection(String title) { - verifier.verify(mock).subSection(title); - return this; - } - - @Override - public Document paragraph(String text) { - verifier.verify(mock).paragraph(text); - return this; - } - - @Override - public Document codeBlock(String description, String code) { - verifier.verify(mock).codeBlock(description, code); - return this; - } - - @Override - public Document table(String description, String table) { - verifier.verify(mock).table(description, table); - return this; - } - - @Override - public Document image(String description, String filePath) { - verifier.verify(mock).image(description, filePath); - return this; - } - } -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/DocTestTests.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/DocTestTests.java deleted file mode 100644 index 704677764d..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/DocTestTests.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.test; - -import org.junit.runner.RunWith; -import org.junit.runners.Suite; - -/** - * Suite to run all doc tests in one shot for local testing - */ -@RunWith(Suite.class) -@Suite.SuiteClasses({ - SqlRequestTest.class, - SqlResponseTest.class, - SqlRequestFormatTest.class, - SqlResponseFormatTest.class, - DocBuilderTest.class, - RstDocumentTest.class, - DataTableTest.class, -}) -public class DocTestTests { -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/RstDocumentTest.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/RstDocumentTest.java deleted file mode 100644 index 3b7faf546e..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/RstDocumentTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.test; - -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; - -import java.io.ByteArrayOutputStream; -import java.io.PrintWriter; -import org.junit.Before; -import org.junit.Test; -import org.opensearch.sql.doctest.core.markup.RstDocument; - -/** - * Test cases for {@link RstDocument} - */ -public class RstDocumentTest { - - private ByteArrayOutputStream content; - - private RstDocument document; - - @Before - public void setUp() { - content = new ByteArrayOutputStream(); - document = new RstDocument(new PrintWriter(content, true)); // Enable auto flush - } - - @Test - public void testSection() { - document.section("Test Section"); - assertThat( - content.toString(), - is( - "Test Section\n" + - "============\n" + - "\n" - ) - ); - } - - @Test - public void testSubSection() { - document.subSection("Test Sub Section"); - assertThat( - content.toString(), - is( - "Test Sub Section\n" + - "----------------\n" + - "\n" - ) - ); - } - - @Test - public void testParagraph() { - document.paragraph("Test paragraph"); - assertThat( - content.toString(), - is( - "Test paragraph\n" + - "\n" - ) - ); - } - - @Test - public void testCodeBlock() { - document.codeBlock("Test code", ">> curl localhost:9200"); - assertThat( - content.toString(), - is( - "Test code::\n" + - "\n" + - "\t>> curl localhost:9200\n" + - "\n" - ) - ); - } - - @Test - public void testTable() { - document.table( - "Test table", - "+----------+\n" + - "|Test Table|\n" + - "+==========+\n" + - "| test data|\n" + - "+----------+" - ); - - assertThat( - content.toString(), - is( - "Test table:\n" + - "\n" + - "+----------+\n" + - "|Test Table|\n" + - "+==========+\n" + - "| test data|\n" + - "+----------+\n" + - "\n" - ) - ); - } - - @Test - public void testImage() { - document.image("Query syntax", "/docs/user/img/query_syntax.png"); - - assertThat( - content.toString(), - is( - "Query syntax:\n" + - "\n" + - ".. image:: /docs/user/img/query_syntax.png\n" + - "\n" - ) - ); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlRequestFormatTest.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlRequestFormatTest.java deleted file mode 100644 index 2cecc4c13e..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlRequestFormatTest.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.test; - -import static org.hamcrest.Matchers.emptyString; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.CURL_REQUEST; -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.OPENSEARCH_DASHBOARD_REQUEST; -import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; - -import org.junit.Test; -import org.opensearch.sql.doctest.core.request.SqlRequest; -import org.opensearch.sql.doctest.core.request.SqlRequest.UrlParam; -import org.opensearch.sql.doctest.core.request.SqlRequestFormat; - -/** - * Test cases for {@link SqlRequestFormat} - */ -public class SqlRequestFormatTest { - - private final SqlRequest sqlRequest = new SqlRequest( - "POST", - QUERY_API_ENDPOINT, - "{\"query\":\"SELECT * FROM accounts\"}", - new UrlParam("format", "jdbc") - ); - - @Test - public void testIgnoreRequestFormat() { - assertThat(IGNORE_REQUEST.format(sqlRequest), emptyString()); - } - - @Test - public void testCurlFormat() { - String expected = - ">> curl -H 'Content-Type: application/json' -X POST localhost:9200/_plugins/_sql?format=jdbc -d '{\n" + - " \"query\" : \"SELECT * FROM accounts\"\n" + - "}'"; - assertThat(CURL_REQUEST.format(sqlRequest), is(expected)); - } - - @Test - public void testOpenSearchDashboardsFormat() { - String expected = - "POST /_plugins/_sql?format=jdbc\n" + - "{\n" + - " \"query\" : \"SELECT * FROM accounts\"\n" + - "}"; - assertThat(OPENSEARCH_DASHBOARD_REQUEST.format(sqlRequest), is(expected)); - } - - @Test - public void multiLineSqlInOpenSearchDashboardRequestShouldBeWellFormatted() { - SqlRequest multiLineSqlRequest = new SqlRequest( - "POST", - "/_plugins/_sql", - "{\"query\":\"SELECT *\\nFROM accounts\\nWHERE age > 30\"}" - ); - - String expected = - "POST /_plugins/_sql\n" + - "{\n" + - " \"query\" : \"\"\"\n" + - "\tSELECT *\n" + - "\tFROM accounts\n" + - "\tWHERE age > 30\n" + - "\t\"\"\"\n" + - "}"; - assertThat(OPENSEARCH_DASHBOARD_REQUEST.format(multiLineSqlRequest), is(expected)); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlRequestTest.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlRequestTest.java deleted file mode 100644 index d459e7aaca..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlRequestTest.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.test; - -import static org.hamcrest.Matchers.hasEntry; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; - -import com.google.common.base.Charsets; -import com.google.common.io.CharStreams; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import org.junit.Test; -import org.mockito.ArgumentCaptor; -import org.opensearch.client.Request; -import org.opensearch.client.RestClient; -import org.opensearch.sql.doctest.core.request.SqlRequest; -import org.opensearch.sql.doctest.core.request.SqlRequest.UrlParam; - -/** - * Test cases for {@link SqlRequest} - */ -public class SqlRequestTest { - - @Test - public void requestShouldIncludeAllFields() throws IOException { - String method = "POST"; - String endpoint = QUERY_API_ENDPOINT; - String body = "{\"query\":\"SELECT * FROM accounts\"}"; - String key = "format"; - String value = "jdbc"; - UrlParam param = new UrlParam(key, value); - - RestClient client = mock(RestClient.class); - SqlRequest sqlRequest = new SqlRequest(method, endpoint, body, param); - sqlRequest.send(client); - - ArgumentCaptor argument = ArgumentCaptor.forClass(Request.class); - verify(client).performRequest(argument.capture()); - Request actual = argument.getValue(); - assertThat(actual.getMethod(), is(method)); - assertThat(actual.getEndpoint(), is(endpoint)); - assertThat(actual.getParameters(), hasEntry(key, value)); - assertThat(body(actual), is(body)); - } - - @Test(expected = IllegalArgumentException.class) - public void badUrlParamShouldThrowException() { - new UrlParam("test"); - } - - private String body(Request request) throws IOException { - InputStream content = request.getEntity().getContent(); - return CharStreams.toString(new InputStreamReader(content, Charsets.UTF_8)); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlResponseFormatTest.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlResponseFormatTest.java deleted file mode 100644 index 689e9f0b98..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlResponseFormatTest.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.test; - -import static org.hamcrest.Matchers.emptyString; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.ORIGINAL_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.TABLE_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.TABLE_UNSORTED_RESPONSE; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import org.apache.http.HttpEntity; -import org.junit.Before; -import org.junit.Test; -import org.opensearch.client.Response; -import org.opensearch.sql.doctest.core.response.SqlResponse; -import org.opensearch.sql.doctest.core.response.SqlResponseFormat; - -/** - * Test cases for {@link SqlResponseFormat} - */ -public class SqlResponseFormatTest { - - private final String expected = - "{" + - "\"schema\":[{\"name\":\"firstname\",\"type\":\"text\"}]," + - "\"datarows\":[[\"John\"]]," + - "\"total\":10," + - "\"size\":1," + - "\"status\":200" + - "}"; - - private SqlResponse sqlResponse; - - @Before - public void setUp() throws IOException { - mockResponse(expected); - } - - @Test - public void testIgnoreResponseFormat() { - assertThat(IGNORE_RESPONSE.format(sqlResponse), emptyString()); - } - - @Test - public void testOriginalFormat() { - assertThat(ORIGINAL_RESPONSE.format(sqlResponse), is(expected + "\n")); - } - - @Test - public void testPrettyJsonFormat() { - assertThat( - PRETTY_JSON_RESPONSE.format(sqlResponse), - is( - "{\n" + - " \"schema\" : [\n" + - " {\n" + - " \"name\" : \"firstname\",\n" + - " \"type\" : \"text\"\n" + - " }\n" + - " ],\n" + - " \"datarows\" : [\n" + - " [\n" + - " \"John\"\n" + - " ]\n" + - " ],\n" + - " \"total\" : 10,\n" + - " \"size\" : 1,\n" + - " \"status\" : 200\n" + - "}" - ) - ); - } - - @Test - public void testTableFormat() { - assertThat( - TABLE_RESPONSE.format(sqlResponse), - is( - "+---------+\n" + - "|firstname|\n" + - "+=========+\n" + - "| John|\n" + - "+---------+\n" - ) - ); - } - - @Test - public void rowsInTableShouldBeSorted() throws IOException { - mockResponse( - "{" + - "\"schema\":[" + - "{\"name\":\"firstname\",\"type\":\"text\"}," + - "{\"name\":\"age\",\"type\":\"integer\"}" + - "]," + - "\"datarows\":[" + - "[\"John\", 30]," + - "[\"John\", 24]," + - "[\"Allen\", 45]" + - "]," + - "\"total\":10," + - "\"size\":3," + - "\"status\":200" + - "}" - ); - - assertThat( - TABLE_RESPONSE.format(sqlResponse), - is( - "+---------+---+\n" + - "|firstname|age|\n" + - "+=========+===+\n" + - "| Allen| 45|\n" + - "+---------+---+\n" + - "| John| 24|\n" + - "+---------+---+\n" + - "| John| 30|\n" + - "+---------+---+\n" - ) - ); - } - - @Test - public void rowsInTableUnsortedShouldMaintainOriginalOrder() throws IOException { - mockResponse( - "{" + - "\"schema\":[" + - "{\"name\":\"firstname\",\"type\":\"text\"}," + - "{\"name\":\"age\",\"type\":\"integer\"}" + - "]," + - "\"datarows\":[" + - "[\"John\", 30]," + - "[\"John\", 24]," + - "[\"Allen\", 45]" + - "]," + - "\"total\":10," + - "\"size\":3," + - "\"status\":200" + - "}" - ); - - assertThat( - TABLE_UNSORTED_RESPONSE.format(sqlResponse), - is( - "+---------+---+\n" + - "|firstname|age|\n" + - "+=========+===+\n" + - "| John| 30|\n" + - "+---------+---+\n" + - "| John| 24|\n" + - "+---------+---+\n" + - "| Allen| 45|\n" + - "+---------+---+\n" - ) - ); - } - - private void mockResponse(String content) throws IOException { - Response response = mock(Response.class); - HttpEntity entity = mock(HttpEntity.class); - when(response.getEntity()).thenReturn(entity); - when(entity.getContent()).thenReturn(new ByteArrayInputStream(content.getBytes())); - sqlResponse = new SqlResponse(response); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlResponseTest.java b/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlResponseTest.java deleted file mode 100644 index 50912ce6e7..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/core/test/SqlResponseTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.core.test; - -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import org.apache.http.HttpEntity; -import org.junit.Test; -import org.opensearch.client.Response; -import org.opensearch.sql.doctest.core.response.SqlResponse; - -/** - * Test cases for {@link SqlResponse} - */ -public class SqlResponseTest { - - @Test - public void responseBodyShouldRetainNewLine() throws IOException { - Response response = mock(Response.class); - HttpEntity entity = mock(HttpEntity.class); - String expected = "123\nabc\n"; - when(response.getEntity()).thenReturn(entity); - when(entity.getContent()).thenReturn(new ByteArrayInputStream(expected.getBytes())); - - SqlResponse sqlResponse = new SqlResponse(response); - String actual = sqlResponse.body(); - assertThat(actual, is(expected)); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/dml/DeleteIT.java b/integ-test/src/test/java/org/opensearch/sql/doctest/dml/DeleteIT.java deleted file mode 100644 index c11bbc3f76..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/dml/DeleteIT.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.dml; - -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.OPENSEARCH_DASHBOARD_REQUEST; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; - -import org.opensearch.sql.doctest.core.DocTest; -import org.opensearch.sql.doctest.core.annotation.DocTestConfig; -import org.opensearch.sql.doctest.core.annotation.Section; - -@DocTestConfig(template = "dml/delete.rst", testData = {"accounts.json"}) -public class DeleteIT extends DocTest { - - @Section(1) - public void delete() { - section( - title("DELETE"), - description( - "``DELETE`` statement deletes documents that satisfy the predicates in ``WHERE`` clause.", - "Note that all documents are deleted in the case of ``WHERE`` clause absent." - ), - images("rdd/singleDeleteStatement.png"), - example( - description( - "The ``datarows`` field in this case shows rows impacted, in other words how many", - "documents were just deleted." - ), - post(multiLine( - "DELETE FROM accounts", - "WHERE age > 30" - )), - queryFormat(OPENSEARCH_DASHBOARD_REQUEST, PRETTY_JSON_RESPONSE), - explainFormat(IGNORE_REQUEST, PRETTY_JSON_RESPONSE) - ) - ); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/dql/BasicQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/doctest/dql/BasicQueryIT.java deleted file mode 100644 index 80b8eb03e4..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/dql/BasicQueryIT.java +++ /dev/null @@ -1,310 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.dql; - -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.OPENSEARCH_DASHBOARD_REQUEST; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.TABLE_UNSORTED_RESPONSE; - -import org.opensearch.sql.doctest.core.DocTest; -import org.opensearch.sql.doctest.core.annotation.DocTestConfig; -import org.opensearch.sql.doctest.core.annotation.Section; -import org.opensearch.sql.doctest.core.builder.Example; -import org.opensearch.sql.doctest.core.builder.Requests; - -/** - * Doc test for basic SELECT query. - */ -@DocTestConfig(template = "dql/basics.rst", testData = {"accounts.json"}) -public class BasicQueryIT extends DocTest { - - @Section(1) - public void select() { - section( - title("SELECT"), - description( - "``SELECT`` clause specifies which fields in OpenSearch index should be retrieved."), - images("rdd/selectElements.png", "rdd/selectElement.png"), - example( - title("Selecting All Fields"), - description( - "You can use ``*`` to fetch all fields in the index which is very convenient when you", - "just want to have a quick look at your data." - ), - post("SELECT * FROM accounts") - ), - example( - title("Selecting Specific Fields"), - description( - "More often you would give specific field name(s) in ``SELECT`` clause to", - "avoid large and unnecessary data retrieved." - ), - post("SELECT firstname, lastname FROM accounts") - ), - example( - title("Using Field Alias"), - description( - "Alias is often used to make your query more readable by giving your field a shorter name." - ), - post("SELECT account_number AS num FROM accounts") - ), - example( - title("Selecting Distinct Fields"), - description( - "``DISTINCT`` is useful when you want to de-duplicate and get unique field value.", - "You can provide one or more field names." - ), - post("SELECT DISTINCT age FROM accounts") - ) - ); - } - - @Section(2) - public void from() { - section( - title("FROM"), - description( - "``FROM`` clause specifies OpenSearch index where the data should be retrieved from.", - "You've seen how to specify a single index in FROM clause in last section. Here we", - "provide examples for more use cases.\n\n" + - "Subquery in ``FROM`` clause is also supported. Please check out the documentation for more details." - ), - images("rdd/tableName.png"), - openSearchDashboardsExample( - title("Using Index Alias"), - description( - "Similarly you can give index in ``FROM`` clause an alias and use it across clauses in query." - ), - post("SELECT acc.account_number FROM accounts acc") - ), - openSearchDashboardsExample( - title("Selecting From Multiple Indices by Index Pattern"), - description( - "Alternatively you can query from multiple indices of similar names by index pattern.", - "This is very convenient for indices created by Logstash index template with date as suffix." - ), - post("SELECT account_number FROM account*") - ), - openSearchDashboardsExample( - title("[Deprecating] Selecting From Specific Index Type"), - description( - "You can also specify type name explicitly though this has been deprecated in", - "later OpenSearch version." - ), - post("SELECT account_number FROM accounts/account") - ) - ); - } - - @Section(3) - public void where() { - section( - title("WHERE"), - description( - "``WHERE`` clause specifies only OpenSearch documents that meet the criteria should be affected.", - "It consists of predicates that uses ``=``, ``<>``, ``>``, ``>=``, ``<``, ``<=``, ``IN``,", - "``BETWEEN``, ``LIKE``, ``IS NULL`` or ``IS NOT NULL``. These predicates can be combined by", - "logical operator ``NOT``, ``AND`` or ``OR`` to build more complex expression.\n\n" + - "For ``LIKE`` and other full text search topics, please refer to Full Text Search documentation.\n\n" + - "Besides SQL query, WHERE clause can also be used in SQL statement such as ``DELETE``. Please refer to", - "Data Manipulation Language documentation for details." - ), - example( - title("Comparison Operators"), - description( - "Basic comparison operators, such as ``=``, ``<>``, ``>``, ``>=``, ``<``, ``<=``, can work for", - "number, string or date.", - "``IN`` and ``BETWEEN`` is convenient for comparison with multiple values or a range." - ), - post(multiLine( - "SELECT account_number", - "FROM accounts", - "WHERE account_number = 1" - )) - ), - example( - title("Missing Fields"), - description( - "As NoSQL database, OpenSearch allows for flexible schema that documents in an index may have", - "different fields. In this case, you can use ``IS NULL`` or ``IS NOT NULL`` to retrieve missing", - "fields or existing fields only.\n\n" + - "Note that for now we don't differentiate missing field and field set to ``NULL`` explicitly." - ), - post(multiLine( - "SELECT account_number, employer", - "FROM accounts", - "WHERE employer IS NULL" - )) - ) - ); - } - - @Section(4) - public void groupBy() { - section( - title("GROUP BY"), - description( - "``GROUP BY`` groups documents with same field value into buckets. It is often used along with", - "aggregation functions to aggregate inside each bucket. Please refer to SQL Functions documentation", - "for more details.\n\n" + - "Note that ``WHERE`` clause is applied before ``GROUP BY`` clause." - ), - example( - title("Grouping by Fields"), - description(), - post(multiLine( - "SELECT age", - "FROM accounts", - "GROUP BY age" - )) - ), - example( - title("Grouping by Field Alias"), - description("Field alias is accessible in ``GROUP BY`` clause."), - post(multiLine( - "SELECT account_number AS num", - "FROM accounts", - "GROUP BY num" - )) - ), - example( - title("Grouping by Ordinal"), - description( - "Alternatively field ordinal in ``SELECT`` clause can be used too. However this is not", - "recommended because your ``GROUP BY`` clause depends on fields in ``SELECT`` clause", - "and require to change accordingly." - ), - post(multiLine( - "SELECT age", - "FROM accounts", - "GROUP BY 1" - )) - ), - example( - title("Grouping by Scalar Function"), - description( - "Scalar function can be used in ``GROUP BY`` clause and it's required to be present in", - "``SELECT`` clause too." - ), - post(multiLine( - "SELECT ABS(age) AS a", - "FROM accounts", - "GROUP BY ABS(age)" - )) - ) - ); - } - - @Section(5) - public void having() { - section( - title("HAVING"), - description( - "``HAVING`` clause filters result from ``GROUP BY`` clause by predicate(s). Because of this,", - "aggregation function, even different from those on ``SELECT`` clause, can be used in predicate." - ), - example( - description(), - post(multiLine( - "SELECT age, MAX(balance)", - "FROM accounts", - "GROUP BY age", - "HAVING MIN(balance) > 10000" - )) - ) - ); - } - - @Section(6) - public void orderBy() { - section( - title("ORDER BY"), - description( - "``ORDER BY`` clause specifies which fields used to sort the result and in which direction."), - orderByExample( - title("Ordering by Fields"), - description( - "Besides regular field names, ordinal, alias or scalar function can also be used similarly", - "as in ``GROUP BY``. ``ASC`` (by default) or ``DESC`` can be appended to indicate sorting in", - "ascending or descending order." - ), - post("SELECT account_number FROM accounts ORDER BY account_number DESC") - ), - orderByExample( - title("Specifying Order for Null"), - description( - "Additionally you can specify if documents with missing field be put first or last.", - "The default behavior of OpenSearch is to return nulls or missing last.", - "You can make them present before non-nulls by using ``IS NOT NULL``." - ), - post(multiLine( - "SELECT employer", - "FROM accounts", - "ORDER BY employer IS NOT NULL" - )) - ) - ); - } - - @Section(7) - public void limit() { - section( - title("LIMIT"), - description( - "Mostly specifying maximum number of documents returned is necessary to prevent fetching", - "large amount of data into memory. `LIMIT` clause is helpful in this case." - ), - example( - title("Limiting Result Size"), - description( - "Given a positive number, ``LIMIT`` uses it as page size to fetch result of that size at most." - ), - post(multiLine( - "SELECT account_number", - "FROM accounts", - "ORDER BY account_number LIMIT 1" - )) - ), - example( - title("Fetching at Offset"), - description( - "Offset position can be given as first argument to indicate where to start fetching.", - "This can be used as simple pagination solution though it's inefficient on large index.", - "Generally ``ORDER BY`` is required in this case to ensure the same order between pages." - ), - post(multiLine( - "SELECT account_number", - "FROM accounts", - "ORDER BY account_number LIMIT 1, 1" - )) - ) - ); - } - - /** - * Document only OpenSearch Dashboard request for example and ignore response as well as explain - */ - private Example openSearchDashboardsExample(String title, String description, Requests requests) { - return example(title, description, requests, - queryFormat(OPENSEARCH_DASHBOARD_REQUEST, IGNORE_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ); - } - - /** - * Example for ORDER BY needs to maintain the order in original result - */ - private Example orderByExample(String title, String description, Requests requests) { - return example(title, description, requests, - queryFormat(OPENSEARCH_DASHBOARD_REQUEST, TABLE_UNSORTED_RESPONSE), - explainFormat(IGNORE_REQUEST, PRETTY_JSON_RESPONSE) - ); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/dql/ComplexQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/doctest/dql/ComplexQueryIT.java deleted file mode 100644 index 22b0b3367d..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/dql/ComplexQueryIT.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.dql; - -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.OPENSEARCH_DASHBOARD_REQUEST; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.TABLE_RESPONSE; - -import org.junit.Ignore; -import org.opensearch.sql.doctest.core.DocTest; -import org.opensearch.sql.doctest.core.annotation.DocTestConfig; -import org.opensearch.sql.doctest.core.annotation.Section; -import org.opensearch.sql.doctest.core.builder.Example; -import org.opensearch.sql.doctest.core.builder.Requests; - -@DocTestConfig(template = "dql/complex.rst", testData = {"accounts.json", "employees_nested.json"}) -public class ComplexQueryIT extends DocTest { - - @Section(1) - public void subquery() { - section( - title("Subquery"), - description( - "A subquery is a complete ``SELECT`` statement which is used within another statement", - "and enclosed in parenthesis. From the explain output, you can notice that some subquery", - "are actually transformed to an equivalent join query to execute." - ), - /* - example( - title("Scalar Value Subquery"), - description( - "" - ), - post( - "SELECT firstname, lastname, balance " + - "FROM accounts " + - "WHERE balance >= ( " + - " SELECT AVG(balance) FROM accounts " + - ") " - ) - ),*/ - example( - title("Table Subquery"), - description(""), - post(multiLine( - "SELECT a1.firstname, a1.lastname, a1.balance", - "FROM accounts a1", - "WHERE a1.account_number IN (", - " SELECT a2.account_number", - " FROM accounts a2", - " WHERE a2.balance > 10000", - ")" - )) - ), - example( - title("Subquery in FROM Clause"), - description(""), - post(multiLine( - "SELECT a.f, a.l, a.a", - "FROM (", - " SELECT firstname AS f, lastname AS l, age AS a", - " FROM accounts", - " WHERE age > 30", - ") AS a" - )) - ) - ); - } - - @Section(2) - public void joins() { - section( - title("JOINs"), - description( - "A ``JOIN`` clause combines columns from one or more indices by using values common to each." - ), - images("rdd/tableSource.png", "rdd/joinPart.png"), - example( - title("Inner Join"), - description( - "Inner join is very commonly used that creates a new result set by combining columns", - "of two indices based on the join predicates specified. It iterates both indices and", - "compare each document to find all that satisfy the join predicates. Keyword ``JOIN``", - "is used and preceded by ``INNER`` keyword optionally. The join predicate(s) is specified", - "by ``ON`` clause.\n\n", - "Remark that the explain API output for join queries looks complicated. This is because", - "a join query is associated with two OpenSearch DSL queries underlying and execute in", - "the separate query planner framework. You can interpret it by looking into the logical", - "plan and physical plan." - ), - post(multiLine( - "SELECT", - " a.account_number, a.firstname, a.lastname,", - " e.id, e.name", - "FROM accounts a", - "JOIN employees_nested e", - " ON a.account_number = e.id" - )) - ), - joinExampleWithoutExplain( - title("Cross Join"), - description( - "Cross join or Cartesian join combines each document from the first index with each from", - "the second. The result set is the Cartesian Product of documents from both indices.", - "It appears to be similar to inner join without ``ON`` clause to specify join condition.\n\n", - "Caveat: It is risky to do cross join even on two indices of medium size. This may trigger", - "our circuit breaker to terminate the query to avoid out of memory issue." - ), - post(multiLine( - "SELECT", - " a.account_number, a.firstname, a.lastname,", - " e.id, e.name", - "FROM accounts a", - "JOIN employees_nested e" - )) - ), - joinExampleWithoutExplain( - title("Outer Join"), - description( - "Outer join is used to retain documents from one or both indices although it does not satisfy", - "join predicate. For now, only ``LEFT OUTER JOIN`` is supported to retain rows from first index.", - "Note that keyword ``OUTER`` is optional." - ), - post(multiLine( - "SELECT", - " a.account_number, a.firstname, a.lastname,", - " e.id, e.name", - "FROM accounts a", - "LEFT JOIN employees_nested e", - " ON a.account_number = e.id" - )) - ) - ); - } - - @Ignore("Multi-query doesn't work for default format: https://github.com/opendistro-for-elasticsearch/sql/issues/388") - @Section(3) - public void setOperations() { - section( - title("Set Operations"), - description( - "Set operations allow results of multiple queries to be combined into a single result set.", - "The results to be combined are required to be of same type. In other word, they require to", - "have same column. Otherwise, a semantic analysis exception is raised." - ), - example( - title("UNION Operator"), - description( - "A ``UNION`` clause combines the results of two queries into a single result set. Duplicate rows", - "are removed unless ``UNION ALL`` clause is being used. A common use case of ``UNION`` is to combine", - "result set from data partitioned in indices daily or monthly." - ), - post(multiLine( - "SELECT balance, firstname, lastname", - "FROM accounts WHERE balance < 10000", - "UNION", - "SELECT balance, firstname, lastname", - "FROM accounts WHERE balance > 30000" - )) - ), - example( - title("MINUS Operator"), - description( - "A ``MINUS`` clause takes two queries too but returns resulting rows of first query that", - "do not appear in the other query. Duplicate rows are removed automatically as well." - ), - post(multiLine( - "SELECT balance, age", - "FROM accounts", - "WHERE balance < 10000", - "MINUS", - "SELECT balance, age", - "FROM accounts", - "WHERE age < 35" - )) - ) - ); - } - - private Example joinExampleWithoutExplain(String title, String description, Requests requests) { - return example(title, description, requests, - queryFormat(OPENSEARCH_DASHBOARD_REQUEST, TABLE_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/dql/MetaDataQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/doctest/dql/MetaDataQueryIT.java deleted file mode 100644 index 6de19690b0..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/dql/MetaDataQueryIT.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.dql; - -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.OPENSEARCH_DASHBOARD_REQUEST; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.TABLE_RESPONSE; - -import org.opensearch.sql.doctest.core.DocTest; -import org.opensearch.sql.doctest.core.annotation.DocTestConfig; -import org.opensearch.sql.doctest.core.annotation.Section; -import org.opensearch.sql.doctest.core.builder.Example; -import org.opensearch.sql.doctest.core.builder.Requests; - -@DocTestConfig(template = "dql/metadata.rst", testData = {"accounts.json", "employees_nested.json"}) -public class MetaDataQueryIT extends DocTest { - - @Section(1) - public void queryMetaData() { - section( - title("Querying Metadata"), - description( - "You can query your indices metadata by ``SHOW`` and ``DESCRIBE`` statement. These commands are", - "very useful for database management tool to enumerate all existing indices and get basic information", - "from the cluster." - ), - images("rdd/showStatement.png", "rdd/showFilter.png"), - metadataQueryExample( - title("Show All Indices Information"), - description( - "``SHOW`` statement lists all indices that match the search pattern. By using wildcard '%',", - "information for all indices in the cluster is returned." - ), - post("SHOW TABLES LIKE %") - ), - metadataQueryExample( - title("Show Specific Index Information"), - description( - "Here is an example that searches metadata for index name prefixed by 'acc'"), - post("SHOW TABLES LIKE acc%") - ), - metadataQueryExample( - title("Describe Index Fields Information"), - description( - "``DESCRIBE`` statement lists all fields for indices that can match the search pattern."), - post("DESCRIBE TABLES LIKE accounts") - ) - ); - } - - /** - * Explain doesn't work for SHOW/DESCRIBE so skip it - */ - private Example metadataQueryExample(String title, String description, Requests requests) { - return example(title, description, requests, - queryFormat(OPENSEARCH_DASHBOARD_REQUEST, TABLE_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/dql/SQLFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/doctest/dql/SQLFunctionsIT.java deleted file mode 100644 index 00b1f11cac..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/dql/SQLFunctionsIT.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.dql; - -import static org.opensearch.sql.legacy.antlr.semantic.types.TypeExpression.TypeExpressionSpec; - -import org.opensearch.sql.doctest.core.DocTest; -import org.opensearch.sql.doctest.core.annotation.DocTestConfig; -import org.opensearch.sql.doctest.core.annotation.Section; -import org.opensearch.sql.legacy.antlr.semantic.types.function.ScalarFunction; -import org.opensearch.sql.legacy.utils.StringUtils; - -@DocTestConfig(template = "dql/functions.rst") -public class SQLFunctionsIT extends DocTest { - - /** - * List only specifications of all SQL functions supported for now - */ - @Section - public void listFunctions() { - for (ScalarFunction func : ScalarFunction - .values()) { // Java Enum.values() return enums in order they are defined - section( - title(func.getName()), - description(listFunctionSpecs(func)) - ); - } - } - - private String listFunctionSpecs(ScalarFunction func) { - TypeExpressionSpec[] specs = func.specifications(); - if (specs.length == 0) { - return "Specification is undefined and type check is skipped for now"; - } - - StringBuilder specStr = new StringBuilder("Specifications: \n\n"); - for (int i = 0; i < specs.length; i++) { - specStr.append( - StringUtils.format("%d. %s%s\n", (i + 1), func.getName(), specs[i]) - ); - } - return specStr.toString(); - } -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/interfaces/EndpointIT.java b/integ-test/src/test/java/org/opensearch/sql/doctest/interfaces/EndpointIT.java deleted file mode 100644 index 00d32e5b95..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/interfaces/EndpointIT.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.interfaces; - -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.CURL_REQUEST; -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; - -import org.opensearch.sql.doctest.core.DocTest; -import org.opensearch.sql.doctest.core.annotation.DocTestConfig; -import org.opensearch.sql.doctest.core.annotation.Section; - -/** - * Doc test for endpoints to access the plugin. - */ -@DocTestConfig(template = "interfaces/endpoint.rst", testData = {"accounts.json"}) -public class EndpointIT extends DocTest { - - @Section(1) - public void queryByPost() { - section( - title("POST"), - description("You can also send HTTP POST request with your query in request body."), - example( - description(), - post("SELECT * FROM accounts"), - queryFormat(CURL_REQUEST, IGNORE_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ) - ); - } - - @Section(2) - public void explainQuery() { - section( - title("Explain"), - description( - "To translate your query, send it to explain endpoint. The explain output is OpenSearch", - "domain specific language (DSL) in JSON format. You can just copy and paste it to your", - "console to run it against OpenSearch directly." - ), - example( - description(), - post("SELECT firstname, lastname FROM accounts WHERE age > 20"), - queryFormat(IGNORE_REQUEST, IGNORE_RESPONSE), - explainFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE) - ) - ); - } - - @Section(3) - public void cursorQuery() { - section( - title("Cursor"), - description( - "To get paginated response for a query, user needs to provide `fetch_size` parameter as part of normal query.", - "The value of `fetch_size` should be greater than `0`. In absence of `fetch_size`, default value of 1000 is used.", - "A value of `0` will fallback to non-paginated response.", - "This feature is only available over `jdbc` format for now." - ), - example( - description(), - post("SELECT firstname, lastname FROM accounts WHERE age > 20 ORDER BY state ASC"), - queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ) - ); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/doctest/interfaces/ProtocolIT.java b/integ-test/src/test/java/org/opensearch/sql/doctest/interfaces/ProtocolIT.java deleted file mode 100644 index 4ebf4899d2..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/doctest/interfaces/ProtocolIT.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.doctest.interfaces; - -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.CURL_REQUEST; -import static org.opensearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.ORIGINAL_RESPONSE; -import static org.opensearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; - -import org.opensearch.sql.doctest.core.DocTest; -import org.opensearch.sql.doctest.core.annotation.DocTestConfig; -import org.opensearch.sql.doctest.core.annotation.Section; - -/** - * Doc test for plugin supported protocols. - */ -@DocTestConfig(template = "interfaces/protocol.rst", testData = {"accounts.json"}) -public class ProtocolIT extends DocTest { - - @Section(1) - public void requestFormat() { - section( - title("Request Format"), - description( - "The body of HTTP POST request can take a few more other fields with SQL query."), - example( - description( - "Use `filter` to add more conditions to OpenSearch DSL directly." - ), - post( - body( - "\"query\": \"SELECT firstname, lastname, balance FROM accounts\"", - "\"filter\":{\"range\":{\"balance\":{\"lt\":10000}}}" - ) - ), - queryFormat(CURL_REQUEST, IGNORE_RESPONSE), - explainFormat(IGNORE_REQUEST, PRETTY_JSON_RESPONSE) - ), - example( - description("Use `parameters` for actual parameter value in prepared SQL query."), - post( - body( - "\"query\": \"SELECT * FROM accounts WHERE age = ?\"", - "\"parameters\": [{\"type\": \"integer\", \"value\": 30}]" - ) - ), - queryFormat(CURL_REQUEST, IGNORE_RESPONSE), - explainFormat(IGNORE_REQUEST, PRETTY_JSON_RESPONSE) - ) - ); - } - - @Section(2) - public void responseInJDBCFormat() { - section( - title("JDBC Format"), - description( - "By default the plugin return JDBC format. JDBC format is provided for JDBC driver and client side that needs both schema and", - "result set well formatted." - ), - example( - description( - "Here is an example for normal response. The `schema` includes field name and its type", - "and `datarows` includes the result set." - ), - post("SELECT firstname, lastname, age FROM accounts ORDER BY age LIMIT 2"), - queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ), - example( - description( - "If any error occurred, error message and the cause will be returned instead."), - post("SELECT unknown FROM accounts", params("format=jdbc")), - queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ) - ); - } - - @Section(3) - public void originalDSLResponse() { - section( - title("OpenSearch DSL"), - description( - "The plugin returns original response from OpenSearch in JSON. Because this is", - "the native response from OpenSearch, extra efforts are needed to parse and interpret it." - ), - example( - description(), - post("SELECT firstname, lastname, age FROM accounts ORDER BY age LIMIT 2", - params("format=json")), - queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ) - ); - } - - @Section(4) - public void responseInCSVFormat() { - section( - title("CSV Format"), - description("You can also use CSV format to download result set as CSV."), - example( - description(), - post("SELECT firstname, lastname, age FROM accounts ORDER BY age", - params("format=csv")), - queryFormat(CURL_REQUEST, ORIGINAL_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ) - ); - } - - @Section(5) - public void responseInRawFormat() { - section( - title("Raw Format"), - description( - "Additionally raw format can be used to pipe the result to other command line tool for post processing." - ), - example( - description(), - post("SELECT firstname, lastname, age FROM accounts ORDER BY age", - params("format=raw")), - queryFormat(CURL_REQUEST, ORIGINAL_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ) - ); - } - -} diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java index 264e601f20..9a08302577 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java @@ -489,28 +489,12 @@ public void percentileAggregationTest() throws Exception { Assert.assertEquals("32.0,32.0,34.0,36.0,38.0,40.0,40.0", lines.get(0)); } - @Ignore("only work for legacy engine") - public void includeTypeAndNotScore() throws Exception { - String query = - String.format(Locale.ROOT, "select age , firstname from %s where age > 31 limit 2", - TEST_INDEX_ACCOUNT); - CSVResult csvResult = executeCsvRequest(query, false, false, true, false); - List headers = csvResult.getHeaders(); - Assert.assertEquals(3, headers.size()); - Assert.assertTrue(headers.contains("age")); - Assert.assertTrue(headers.contains("firstname")); - Assert.assertTrue(headers.contains("_type")); - List lines = csvResult.getLines(); - Assert.assertTrue(lines.get(0).contains(",_doc") || lines.get(0).contains("_doc,")); - Assert.assertTrue(lines.get(1).contains(",_doc") || lines.get(1).contains("_doc,")); - } - @Test - public void includeScoreAndNotType() throws Exception { + public void includeScore() throws Exception { String query = String.format(Locale.ROOT, "select age , firstname from %s where age > 31 order by _score desc limit 2 ", TEST_INDEX_ACCOUNT); - CSVResult csvResult = executeCsvRequest(query, false, true, false, false); + CSVResult csvResult = executeCsvRequest(query, false, true, false); List headers = csvResult.getHeaders(); Assert.assertEquals(3, headers.size()); Assert.assertTrue(headers.contains("age")); @@ -521,24 +505,6 @@ public void includeScoreAndNotType() throws Exception { Assert.assertTrue(lines.get(1).contains("1.0")); } - @Test - public void includeScoreAndType() throws Exception { - String query = String.format(Locale.ROOT, - "select age , firstname from %s where age > 31 order by _score desc limit 2 ", - TEST_INDEX_ACCOUNT); - CSVResult csvResult = executeCsvRequest(query, false, true, true, false); - List headers = csvResult.getHeaders(); - Assert.assertEquals(4, headers.size()); - Assert.assertTrue(headers.contains("age")); - Assert.assertTrue(headers.contains("firstname")); - Assert.assertTrue(headers.contains("_score")); - Assert.assertTrue(headers.contains("_type")); - List lines = csvResult.getLines(); - String firstLine = lines.get(0); - Assert.assertTrue(firstLine.contains("_doc,1.0") || firstLine.contains("1.0,_doc")); - Assert.assertTrue(lines.get(1).contains("_doc,1.0") || lines.get(1).contains("1.0,_doc")); - } - /* todo: more tests: * filter/nested and than metric * histogram @@ -584,7 +550,7 @@ public void twoCharsSeperator() throws Exception { public void includeIdAndNotTypeOrScore() throws Exception { String query = String.format(Locale.ROOT, "select age , firstname from %s where lastname = 'Marquez' ", TEST_INDEX_ACCOUNT); - CSVResult csvResult = executeCsvRequest(query, false, false, false, true); + CSVResult csvResult = executeCsvRequest(query, false, false, true); List headers = csvResult.getHeaders(); Assert.assertEquals(3, headers.size()); Assert.assertTrue(headers.contains("age")); @@ -593,21 +559,6 @@ public void includeIdAndNotTypeOrScore() throws Exception { List lines = csvResult.getLines(); Assert.assertTrue(lines.get(0).contains(",437") || lines.get(0).contains("437,")); } - - @Ignore("only work for legacy engine") - public void includeIdAndTypeButNoScore() throws Exception { - String query = String.format(Locale.ROOT, - "select age , firstname from %s where lastname = 'Marquez' ", TEST_INDEX_ACCOUNT); - CSVResult csvResult = executeCsvRequest(query, false, false, true, true); - List headers = csvResult.getHeaders(); - Assert.assertEquals(4, headers.size()); - Assert.assertTrue(headers.contains("age")); - Assert.assertTrue(headers.contains("firstname")); - Assert.assertTrue(headers.contains("_id")); - Assert.assertTrue(headers.contains("_type")); - List lines = csvResult.getLines(); - Assert.assertTrue(lines.get(0).contains("_doc,437") || lines.get(0).contains("437,_doc")); - } //endregion Tests migrated from CSVResultsExtractorTests @Ignore("only work for legacy engine") @@ -622,7 +573,7 @@ public void sensitiveCharacterSanitizeTest() throws IOException { request.setJsonEntity(requestBody); TestUtils.performRequest(client(), request); - CSVResult csvResult = executeCsvRequest("SELECT * FROM userdata", false, false, false, false); + CSVResult csvResult = executeCsvRequest("SELECT * FROM userdata", false, false, false); List headers = csvResult.getHeaders(); Assert.assertEquals(2, headers.size()); Assert.assertTrue(headers.contains("'=cmd|' /C notepad'!_xlbgnm.A1")); @@ -647,7 +598,7 @@ public void sensitiveCharacterSanitizeAndQuotedTest() throws IOException { request.setJsonEntity(requestBody); TestUtils.performRequest(client(), request); - CSVResult csvResult = executeCsvRequest("SELECT * FROM userdata2", false, false, false, false); + CSVResult csvResult = executeCsvRequest("SELECT * FROM userdata2", false, false, false); String headers = String.join(",", csvResult.getHeaders()); Assert.assertTrue(headers.contains("\"'=cmd|' /C notepad'!_xlbgnm.A1,,\"")); Assert.assertTrue(headers.contains("\",@cmd|' /C notepad'!_xlbgnm.A1\"")); @@ -676,7 +627,7 @@ public void sanitizeTest() throws IOException { @Test public void selectFunctionAsFieldTest() throws IOException { String query = "select log(age) from " + TEST_INDEX_ACCOUNT; - CSVResult result = executeCsvRequest(query, false, false, false, false); + CSVResult result = executeCsvRequest(query, false, false, false); List headers = result.getHeaders(); Assert.assertEquals(1, headers.size()); } @@ -706,16 +657,16 @@ private void setFlatOption(boolean flat) { private CSVResult executeCsvRequest(final String query, boolean flat) throws IOException { - return executeCsvRequest(query, flat, false, false, false); + return executeCsvRequest(query, flat, false, false); } private CSVResult executeCsvRequest(final String query, boolean flat, boolean includeScore, - boolean includeType, boolean includeId) throws IOException { + boolean includeId) throws IOException { final String requestBody = super.makeRequest(query); final String endpoint = String.format(Locale.ROOT, - "/_plugins/_sql?format=csv&flat=%b&_id=%b&_score=%b&_type=%b", - flat, includeId, includeScore, includeType); + "/_plugins/_sql?format=csv&flat=%b&_id=%b&_score=%b", + flat, includeId, includeScore); final Request sqlRequest = new Request("POST", endpoint); sqlRequest.setJsonEntity(requestBody); RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/CustomExternalTestCluster.java b/integ-test/src/test/java/org/opensearch/sql/legacy/CustomExternalTestCluster.java deleted file mode 100644 index 5e388d07e1..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/CustomExternalTestCluster.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.legacy; - -import static org.opensearch.action.admin.cluster.node.info.NodesInfoRequest.Metric.HTTP; -import static org.opensearch.action.admin.cluster.node.info.NodesInfoRequest.Metric.SETTINGS; -import static org.opensearch.test.OpenSearchTestCase.getTestTransportType; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.concurrent.atomic.AtomicInteger; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.action.admin.cluster.node.info.NodeInfo; -import org.opensearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.opensearch.client.Client; -import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.network.NetworkModule; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.transport.TransportAddress; -import org.opensearch.env.Environment; -import org.opensearch.http.HttpInfo; -import org.opensearch.plugins.Plugin; -import org.opensearch.test.InternalTestCluster; -import org.opensearch.test.TestCluster; -import org.opensearch.transport.MockTransportClient; -import org.opensearch.transport.nio.MockNioTransportPlugin; - -public class CustomExternalTestCluster extends TestCluster { - - private static final Logger logger = LogManager.getLogger(CustomExternalTestCluster.class); - - private static final AtomicInteger counter = new AtomicInteger(); - public static final String EXTERNAL_CLUSTER_PREFIX = "external_"; - - private final MockTransportClient client; - - private final InetSocketAddress[] httpAddresses; - - private final String clusterName; - - private final int numDataNodes; - private final int numMasterAndDataNodes; - - public CustomExternalTestCluster(Path tempDir, Settings additionalSettings, - Collection> pluginClasses, - TransportAddress... transportAddresses) { - super(0); - Settings.Builder clientSettingsBuilder = Settings.builder() - .put(additionalSettings) - .put("node.name", InternalTestCluster.TRANSPORT_CLIENT_PREFIX + EXTERNAL_CLUSTER_PREFIX + - counter.getAndIncrement()) - .put("client.transport.ignore_cluster_name", true) - .put(Environment.PATH_HOME_SETTING.getKey(), tempDir); - boolean addMockTcpTransport = additionalSettings.get(NetworkModule.TRANSPORT_TYPE_KEY) == null; - - if (addMockTcpTransport) { - String transport = getTestTransportType(); - clientSettingsBuilder.put(NetworkModule.TRANSPORT_TYPE_KEY, transport); - if (pluginClasses.contains(MockNioTransportPlugin.class) == false) { - pluginClasses = new ArrayList<>(pluginClasses); - if (transport.equals(MockNioTransportPlugin.MOCK_NIO_TRANSPORT_NAME)) { - pluginClasses.add(MockNioTransportPlugin.class); - } - } - } - Settings clientSettings = clientSettingsBuilder.build(); - MockTransportClient client = new MockTransportClient(clientSettings, pluginClasses); - try { - client.addTransportAddresses(transportAddresses); - NodesInfoResponse nodeInfos = - client.admin().cluster().prepareNodesInfo().clear() - .addMetrics(SETTINGS.metricName(), HTTP.metricName()) - .get(); - httpAddresses = new InetSocketAddress[nodeInfos.getNodes().size()]; - this.clusterName = nodeInfos.getClusterName().value(); - int dataNodes = 0; - int masterAndDataNodes = 0; - for (int i = 0; i < nodeInfos.getNodes().size(); i++) { - NodeInfo nodeInfo = nodeInfos.getNodes().get(i); - httpAddresses[i] = nodeInfo.getInfo(HttpInfo.class).address().publishAddress().address(); - if (DiscoveryNode.isDataNode(nodeInfo.getSettings())) { - dataNodes++; - masterAndDataNodes++; - } else if (DiscoveryNode.isMasterNode(nodeInfo.getSettings())) { - masterAndDataNodes++; - } - } - this.numDataNodes = dataNodes; - this.numMasterAndDataNodes = masterAndDataNodes; - this.client = client; - - logger.info("Setup ExternalTestCluster [{}] made of [{}] nodes", - nodeInfos.getClusterName().value(), size()); - } catch (Exception e) { - client.close(); - throw e; - } - } - - @Override - public void afterTest() { - - } - - @Override - public Client client() { - return client; - } - - @Override - public int size() { - return httpAddresses.length; - } - - @Override - public int numDataNodes() { - return numDataNodes; - } - - @Override - public int numDataAndMasterNodes() { - return numMasterAndDataNodes; - } - - @Override - public InetSocketAddress[] httpAddresses() { - return httpAddresses; - } - - @Override - public void close() throws IOException { - client.close(); - } - - /** - * This custom ExternalCluster class has ensureEstimatedStats() emptied out to prevent the transport client error - * made from making a request using a closed client after the tests are complete. - */ - @Override - public void ensureEstimatedStats() { - } - - @Override - public Iterable getClients() { - return Collections.singleton(client); - } - - @Override - public NamedWriteableRegistry getNamedWriteableRegistry() { - return client.getNamedWriteableRegistry(); - } - - @Override - public String getClusterName() { - return clusterName; - } -} diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index 7b2d26bd82..8f8ee4a70f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -264,8 +264,6 @@ public static void loadBulk(Client client, String jsonPath, String defaultIndex) IndexRequest indexRequest = new IndexRequest(); indexRequest.index(defaultIndex); - String docType = actionJson.getJSONObject("index").getString("_type"); - indexRequest.type(docType); if (actionJson.getJSONObject("index").has("_id")) { String docId = actionJson.getJSONObject("index").getString("_id"); indexRequest.id(docId); diff --git a/integ-test/src/test/java/org/opensearch/sql/util/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/util/TestUtils.java index fafc962f4a..bd75ead43b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/TestUtils.java @@ -710,8 +710,6 @@ public static void loadBulk(Client client, String jsonPath, String defaultIndex) IndexRequest indexRequest = new IndexRequest(); indexRequest.index(defaultIndex); - String docType = actionJson.getJSONObject("index").getString("_type"); - indexRequest.type(docType); if (actionJson.getJSONObject("index").has("_id")) { String docId = actionJson.getJSONObject("index").getString("_id"); indexRequest.id(docId); diff --git a/legacy/build.gradle b/legacy/build.gradle index 06c694f25d..8cd146ff1c 100644 --- a/legacy/build.gradle +++ b/legacy/build.gradle @@ -79,31 +79,31 @@ test { } dependencies { - compile group: 'com.alibaba', name: 'druid', version:'1.0.15' - compile group: 'org.locationtech.spatial4j', name: 'spatial4j', version:'0.7' - compile group: "org.opensearch.plugin", name: 'parent-join-client', version: "${opensearch_version}" - compile group: "org.opensearch.plugin", name: 'reindex-client', version: "${opensearch_version}" + implementation group: 'com.alibaba', name: 'druid', version:'1.0.15' + implementation group: 'org.locationtech.spatial4j', name: 'spatial4j', version:'0.7' + implementation group: "org.opensearch.plugin", name: 'parent-join-client', version: "${opensearch_version}" + implementation group: "org.opensearch.plugin", name: 'reindex-client', version: "${opensearch_version}" constraints { implementation('commons-codec:commons-codec:1.13') { because 'https://www.whitesourcesoftware.com/vulnerability-database/WS-2019-0379' } } implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' - compile group: 'org.json', name: 'json', version:'20180813' - compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.10' - compile group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" - compile project(':sql') - compile project(':common') - compile project(':opensearch') + implementation group: 'org.json', name: 'json', version:'20180813' + implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.10' + implementation group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" + api project(':sql') + api project(':common') + api project(':opensearch') // ANTLR gradle plugin and runtime dependency antlr "org.antlr:antlr4:4.7.1" - compile "org.antlr:antlr4-runtime:4.7.1" + implementation "org.antlr:antlr4-runtime:4.7.1" compileOnly group: 'javax.servlet', name: 'servlet-api', version:'2.5' - testCompile group: 'org.hamcrest', name: 'hamcrest-core', version:'2.2' - testCompile group: 'org.mockito', name: 'mockito-inline', version:'3.5.0' - testCompile group: 'junit', name: 'junit', version: '4.13.2' - testCompile group: "org.opensearch.client", name: 'transport', version: "${opensearch_version}" + testImplementation group: 'org.hamcrest', name: 'hamcrest-core', version:'2.2' + testImplementation group: 'org.mockito', name: 'mockito-inline', version:'3.5.0' + testImplementation group: 'junit', name: 'junit', version: '4.13.2' + testImplementation group: "org.opensearch.client", name: 'transport', version: "${opensearch_version}" } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/antlr/semantic/visitor/OpenSearchMappingLoader.java b/legacy/src/main/java/org/opensearch/sql/legacy/antlr/semantic/visitor/OpenSearchMappingLoader.java index 460b363804..7bfca0a015 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/antlr/semantic/visitor/OpenSearchMappingLoader.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/antlr/semantic/visitor/OpenSearchMappingLoader.java @@ -9,6 +9,7 @@ import static org.opensearch.sql.legacy.antlr.semantic.types.base.OpenSearchIndex.IndexType.INDEX; import static org.opensearch.sql.legacy.antlr.semantic.types.base.OpenSearchIndex.IndexType.NESTED_FIELD; +import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -172,9 +173,7 @@ private boolean isNotNested(String indexName) { private Set getFieldMappings(String indexName) { IndexMappings indexMappings = clusterState.getFieldMappings(new String[]{indexName}); - Set fieldMappingsSet = indexMappings.allMappings().stream(). - flatMap(typeMappings -> typeMappings.allMappings().stream()). - collect(Collectors.toSet()); + Set fieldMappingsSet = new HashSet<>(indexMappings.allMappings()); for (FieldMappings fieldMappings : fieldMappingsSet) { int size = fieldMappings.data().size(); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/esdomain/LocalClusterState.java b/legacy/src/main/java/org/opensearch/sql/legacy/esdomain/LocalClusterState.java index 3e6746d44d..37d9322b46 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/esdomain/LocalClusterState.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/esdomain/LocalClusterState.java @@ -45,10 +45,6 @@ public class LocalClusterState { private static final Logger LOG = LogManager.getLogger(); - /** - * Default types and field filter to match all - */ - private static final String[] ALL_TYPES = new String[0]; private static final Function> ALL_FIELDS = (anyIndex -> (anyField -> true)); /** @@ -72,7 +68,7 @@ public class LocalClusterState { * Thread-safe mapping cache to save the computation of sourceAsMap() which is not lightweight as thought * Array cannot be used as key because hashCode() always return reference address, so either use wrapper or List. */ - private final Cache, List>, IndexMappings> cache; + private final Cache, IndexMappings> cache; /** * Latest setting value for each registered key. Thread-safe is required. @@ -148,14 +144,7 @@ public T getSettingValue(Settings.Key key) { * Get field mappings by index expressions. All types and fields are included in response. */ public IndexMappings getFieldMappings(String[] indices) { - return getFieldMappings(indices, ALL_TYPES, ALL_FIELDS); - } - - /** - * Get field mappings by index expressions, type. All fields are included in response. - */ - public IndexMappings getFieldMappings(String[] indices, String[] types) { - return getFieldMappings(indices, types, ALL_FIELDS); + return getFieldMappings(indices, ALL_FIELDS); } /** @@ -167,12 +156,10 @@ public IndexMappings getFieldMappings(String[] indices, String[] types) { * to ClusterService.state() here. * * @param indices index name expression - * @param types type name * @param fieldFilter field filter predicate * @return index mapping(s) */ - public IndexMappings getFieldMappings(String[] indices, String[] types, - Function> fieldFilter) { + private IndexMappings getFieldMappings(String[] indices, Function> fieldFilter) { Objects.requireNonNull(clusterService, "Cluster service is null"); Objects.requireNonNull(resolver, "Index name expression resolver is null"); @@ -182,9 +169,9 @@ public IndexMappings getFieldMappings(String[] indices, String[] types, IndexMappings mappings; if (fieldFilter == ALL_FIELDS) { - mappings = findMappingsInCache(state, concreteIndices, types); + mappings = findMappingsInCache(state, concreteIndices); } else { - mappings = findMappings(state, concreteIndices, types, fieldFilter); + mappings = findMappings(state, concreteIndices, fieldFilter); } LOG.debug("Found mappings: {}", mappings); @@ -194,7 +181,7 @@ public IndexMappings getFieldMappings(String[] indices, String[] types, } catch (Exception e) { throw new IllegalStateException( "Failed to read mapping in cluster state for indices=" - + Arrays.toString(indices) + ", types=" + Arrays.toString(types), e); + + Arrays.toString(indices) , e); } } @@ -208,20 +195,19 @@ private String[] resolveIndexExpression(ClusterState state, String[] indices) { return concreteIndices; } - private IndexMappings findMappings(ClusterState state, String[] indices, String[] types, + private IndexMappings findMappings(ClusterState state, String[] indices, Function> fieldFilter) throws IOException { LOG.debug("Cache didn't help. Load and parse mapping in cluster state"); return new IndexMappings( - state.metadata().findMappings(indices, types, fieldFilter) + state.metadata().findMappings(indices, fieldFilter) ); } - private IndexMappings findMappingsInCache(ClusterState state, String[] indices, String[] types) + private IndexMappings findMappingsInCache(ClusterState state, String[] indices) throws ExecutionException { LOG.debug("Looking for mapping in cache: {}", cache.asMap()); - return cache.get( - new Tuple<>(sortToList(indices), sortToList(types)), - () -> findMappings(state, indices, types, ALL_FIELDS) + return cache.get(sortToList(indices), + () -> findMappings(state, indices, ALL_FIELDS) ); } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/esdomain/mapping/IndexMappings.java b/legacy/src/main/java/org/opensearch/sql/legacy/esdomain/mapping/IndexMappings.java index a25542bf88..73eafe741d 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/esdomain/mapping/IndexMappings.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/esdomain/mapping/IndexMappings.java @@ -13,6 +13,7 @@ import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.collect.ImmutableOpenMap; +import org.opensearch.sql.legacy.domain.Field; /** * Index mappings in the cluster. @@ -32,14 +33,14 @@ * ((Map) client.admin().indices().getFieldMappings(request).actionGet().mappings().get("bank") * .get("account").get("balance").sourceAsMap().get("balance")).get("type") */ -public class IndexMappings implements Mappings { +public class IndexMappings implements Mappings { public static final IndexMappings EMPTY = new IndexMappings(); /** - * Mapping from Index name to mappings of all Types in it + * Mapping from Index name to mappings of all fields in it */ - private final Map indexMappings; + private final Map indexMappings; public IndexMappings() { this.indexMappings = emptyMap(); @@ -47,15 +48,15 @@ public IndexMappings() { public IndexMappings(Metadata metaData) { this.indexMappings = buildMappings(metaData.indices(), - indexMetaData -> new TypeMappings(indexMetaData.getMappings())); + indexMetaData -> new FieldMappings(indexMetaData.getMappings().valuesIt().next())); } - public IndexMappings(ImmutableOpenMap> mappings) { - this.indexMappings = buildMappings(mappings, TypeMappings::new); + public IndexMappings(ImmutableOpenMap mappings) { + this.indexMappings = buildMappings(mappings, FieldMappings::new); } @Override - public Map data() { + public Map data() { return indexMappings; } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/esdomain/mapping/TypeMappings.java b/legacy/src/main/java/org/opensearch/sql/legacy/esdomain/mapping/TypeMappings.java deleted file mode 100644 index 8eddf83adb..0000000000 --- a/legacy/src/main/java/org/opensearch/sql/legacy/esdomain/mapping/TypeMappings.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.legacy.esdomain.mapping; - -import java.util.Map; -import java.util.Objects; -import org.opensearch.cluster.metadata.MappingMetadata; -import org.opensearch.common.collect.ImmutableOpenMap; - -/** - * Type mappings in a specific index. - *

- * Sample: - * typeMappings: { - * '_doc': fieldMappings - * } - */ -public class TypeMappings implements Mappings { - - /** - * Mapping from Type name to mappings of all Fields in it - */ - private final Map typeMappings; - - public TypeMappings(ImmutableOpenMap mappings) { - typeMappings = buildMappings(mappings, FieldMappings::new); - } - - @Override - public Map data() { - return typeMappings; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - TypeMappings that = (TypeMappings) o; - return Objects.equals(typeMappings, that.typeMappings); - } - - @Override - public int hashCode() { - return Objects.hash(typeMappings); - } - - @Override - public String toString() { - return "TypeMappings{" + typeMappings + '}'; - } -} diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/GetIndexRequestRestListener.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/GetIndexRequestRestListener.java index b686fa0ac9..b930044345 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/GetIndexRequestRestListener.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/GetIndexRequestRestListener.java @@ -9,6 +9,8 @@ import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import java.io.IOException; import java.util.List; +import java.util.Map; + import org.opensearch.action.admin.indices.get.GetIndexRequest; import org.opensearch.action.admin.indices.get.GetIndexResponse; import org.opensearch.cluster.metadata.AliasMetadata; @@ -23,6 +25,7 @@ import org.opensearch.rest.RestStatus; import org.opensearch.rest.action.RestBuilderListener; import org.opensearch.sql.legacy.antlr.semantic.SemanticAnalysisException; +import org.opensearch.sql.legacy.domain.Field; /** * Created by Eliran on 6/10/2015. @@ -78,18 +81,6 @@ private void writeAliases(List aliases, XContentBuilder builder, builder.endObject(); } - private void writeMappings(ImmutableOpenMap mappings, - XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.startObject(Fields.MAPPINGS); - if (mappings != null) { - for (ObjectObjectCursor typeEntry : mappings) { - builder.field(typeEntry.key); - builder.map(typeEntry.value.sourceAsMap()); - } - } - builder.endObject(); - } - private void writeSettings(Settings settings, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(Fields.SETTINGS); @@ -97,6 +88,14 @@ private void writeSettings(Settings settings, XContentBuilder builder, ToXConten builder.endObject(); } + private void writeMappings(MappingMetadata mappingMetadata, + XContentBuilder builder, ToXContent.Params params) throws IOException { + if ( mappingMetadata != null) { + builder.field(Fields.MAPPINGS); + builder.map(mappingMetadata.getSourceAsMap()); + } + } + static class Fields { static final String ALIASES = "aliases"; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/csv/CSVResultRestExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/csv/CSVResultRestExecutor.java index 7c8cd23ce5..2f11d2c606 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/csv/CSVResultRestExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/csv/CSVResultRestExecutor.java @@ -49,11 +49,10 @@ public String execute(final Client client, final Map params, fin final boolean flat = getBooleanOrDefault(params, "flat", false); final boolean includeScore = getBooleanOrDefault(params, "_score", false); - final boolean includeType = getBooleanOrDefault(params, "_type", false); final boolean includeId = getBooleanOrDefault(params, "_id", false); final List fieldNames = queryAction.getFieldNames().orElse(null); - final CSVResult result = new CSVResultsExtractor(includeScore, includeType, includeId) + final CSVResult result = new CSVResultsExtractor(includeScore, includeId) .extractResults(queryResult, flat, separator, fieldNames); return buildString(separator, result, newLine); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/csv/CSVResultsExtractor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/csv/CSVResultsExtractor.java index 84f11867f6..5a16a9ab61 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/csv/CSVResultsExtractor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/csv/CSVResultsExtractor.java @@ -36,14 +36,12 @@ */ public class CSVResultsExtractor { - private final boolean includeType; private final boolean includeScore; private final boolean includeId; private int currentLineIndex; - public CSVResultsExtractor(boolean includeScore, boolean includeType, boolean includeId) { + public CSVResultsExtractor(boolean includeScore, boolean includeId) { this.includeScore = includeScore; - this.includeType = includeType; this.includeId = includeId; this.currentLineIndex = 0; } @@ -296,9 +294,6 @@ private List createHeadersAndFillDocsMap(final boolean flat, final Searc if (this.includeScore) { doc.put("_score", hit.getScore()); } - if (this.includeType) { - doc.put("_type", hit.getType()); - } // select function as field is a special case where each hit has non-null field (function) // and sourceAsMap is all columns in index (the same as 'SELECT *') diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatter.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatter.java index ea601bb747..aa803975df 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatter.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DateFieldFormatter.java @@ -22,7 +22,6 @@ import org.apache.logging.log4j.Logger; import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.esdomain.mapping.FieldMappings; -import org.opensearch.sql.legacy.esdomain.mapping.TypeMappings; /** * Formatter to transform date fields into a consistent format for consumption by clients. @@ -112,11 +111,10 @@ private Map> getDateFieldFormatMap(String indexName) { Map> formatMap = new HashMap<>(); String[] indices = indexName.split("\\|"); - Collection typeProperties = state.getFieldMappings(indices) + Collection typeProperties = state.getFieldMappings(indices) .allMappings(); - for (TypeMappings mappings: typeProperties) { - FieldMappings fieldMappings = mappings.firstMapping(); + for (FieldMappings fieldMappings: typeProperties) { for (Map.Entry> field : fieldMappings.data().entrySet()) { String fieldName = field.getKey(); Map properties = field.getValue(); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DescribeResultSet.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DescribeResultSet.java index 99ef26f41a..7d09aa5996 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DescribeResultSet.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/DescribeResultSet.java @@ -79,20 +79,14 @@ private List loadColumns() { private List loadRows() { List rows = new ArrayList<>(); GetIndexResponse indexResponse = (GetIndexResponse) queryResult; - ImmutableOpenMap> indexMappings = indexResponse.getMappings(); + ImmutableOpenMap indexMappings = indexResponse.getMappings(); // Iterate through indices in indexMappings - for (ObjectObjectCursor> indexCursor : indexMappings) { + for (ObjectObjectCursor indexCursor : indexMappings) { String index = indexCursor.key; if (matchesPatternIfRegex(index, statement.getIndexPattern())) { - ImmutableOpenMap typeMapping = indexCursor.value; - // Assuming OpenSearch 6.x, iterate through the only type of the index to get mapping data - for (ObjectObjectCursor typeCursor : typeMapping) { - MappingMetadata mappingMetaData = typeCursor.value; - // Load rows for each field in the mapping - rows.addAll(loadIndexData(index, mappingMetaData.getSourceAsMap())); - } + rows.addAll(loadIndexData(index, indexCursor.value.getSourceAsMap())); } } return rows; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java index 36e42d22cc..811d444a7f 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java @@ -206,13 +206,12 @@ protected void addUnmatchedResults(List combinedResults, protected SearchHit createUnmachedResult(List secondTableReturnedFields, int docId, String t1Alias, String t2Alias, SearchHit hit) { String unmatchedId = hit.getId() + "|0"; - Text unamatchedType = new Text(hit.getType() + "|null"); Map documentFields = new HashMap<>(); Map metaFields = new HashMap<>(); hit.getFields().forEach((fieldName, docField) -> (MapperService.META_FIELDS_BEFORE_7DOT8.contains(fieldName) ? metaFields : documentFields).put(fieldName, docField)); - SearchHit searchHit = new SearchHit(docId, unmatchedId, unamatchedType, documentFields, metaFields); + SearchHit searchHit = new SearchHit(docId, unmatchedId, documentFields, metaFields); searchHit.sourceRef(hit.getSourceRef()); searchHit.getSourceAsMap().clear(); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticUtils.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticUtils.java index 71898a4a60..c3b03c653a 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticUtils.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticUtils.java @@ -66,7 +66,6 @@ public static String hitsAsStringResult(SearchHits results, MetaSearchResult met for (SearchHit hit : results) { HashMap value = new HashMap<>(); value.put("_id", hit.getId()); - value.put("_type", hit.getType()); value.put("_score", hit.getScore()); value.put("_source", hit.getSourceAsMap()); searchHits[i] = value; @@ -147,9 +146,6 @@ private static void toXContent(XContentBuilder builder, Params params, List createCombinedResults(TableInJoinRequestBuilder secondTa matchingHit.getFields().forEach((fieldName, docField) -> (MapperService.META_FIELDS_BEFORE_7DOT8.contains(fieldName) ? metaFields : documentFields).put(fieldName, docField)); SearchHit searchHit = new SearchHit(matchingHit.docId(), combinedId, - new Text(matchingHit.getType() + "|" + secondTableHit.getType()), documentFields, metaFields); searchHit.sourceRef(matchingHit.getSourceRef()); searchHit.getSourceAsMap().clear(); @@ -239,7 +238,7 @@ private void createKeyToResultsAndFillOptimizationStructure( Map metaFields = new HashMap<>(); hit.getFields().forEach((fieldName, docField) -> (MapperService.META_FIELDS_BEFORE_7DOT8.contains(fieldName) ? metaFields : documentFields).put(fieldName, docField)); - SearchHit searchHit = new SearchHit(resultIds, hit.getId(), new Text(hit.getType()), documentFields + SearchHit searchHit = new SearchHit(resultIds, hit.getId(), documentFields , metaFields); searchHit.sourceRef(hit.getSourceRef()); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java index 6bbe1ff7fc..cc6971a220 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java @@ -170,8 +170,7 @@ private SearchHit getMergedHit(int currentCombinedResults, String t1Alias, Strin matchedHit.getFields().forEach((fieldName, docField) -> (MapperService.META_FIELDS_BEFORE_7DOT8.contains(fieldName) ? metaFields : documentFields).put(fieldName, docField)); SearchHit searchHit = new SearchHit(currentCombinedResults, hitFromFirstTable.getId() + "|" - + matchedHit.getId(), new Text(hitFromFirstTable.getType() + "|" + matchedHit.getType()), - documentFields, metaFields); + + matchedHit.getId(), documentFields, metaFields); searchHit.sourceRef(hitFromFirstTable.getSourceRef()); searchHit.getSourceAsMap().clear(); searchHit.getSourceAsMap().putAll(hitFromFirstTable.getSourceAsMap()); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java index ae79b6a318..8d37cb0656 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java @@ -129,7 +129,7 @@ private void fillMinusHitsFromOneField(String fieldName, Set fieldValues Map metaFields = new HashMap<>(); someHit.getFields().forEach((field, docField) -> (MapperService.META_FIELDS_BEFORE_7DOT8.contains(field) ? metaFields : documentFields).put(field, docField)); - SearchHit searchHit = new SearchHit(currentId, currentId + "", new Text(someHit.getType()), + SearchHit searchHit = new SearchHit(currentId, currentId + "", documentFields, metaFields); searchHit.sourceRef(someHit.getSourceRef()); searchHit.getSourceAsMap().clear(); @@ -155,7 +155,7 @@ private void fillMinusHitsFromResults(Set comperableHitResu Map metaFields = new HashMap<>(); originalHit.getFields().forEach((fieldName, docField) -> (MapperService.META_FIELDS_BEFORE_7DOT8.contains(fieldName) ? metaFields : documentFields).put(fieldName, docField)); - SearchHit searchHit = new SearchHit(currentId, originalHit.getId(), new Text(originalHit.getType()), + SearchHit searchHit = new SearchHit(currentId, originalHit.getId(), documentFields, metaFields); searchHit.sourceRef(originalHit.getSourceRef()); searchHit.getSourceAsMap().clear(); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/UnionExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/UnionExecutor.java index 86ef510bc2..fba2b2dcaf 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/UnionExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/UnionExecutor.java @@ -61,7 +61,7 @@ private void fillInternalSearchHits(List unionHits, SearchHit[] hits, Map metaFields = new HashMap<>(); hit.getFields().forEach((fieldName, docField) -> (MapperService.META_FIELDS_BEFORE_7DOT8.contains(fieldName) ? metaFields : documentFields).put(fieldName, docField)); - SearchHit searchHit = new SearchHit(currentId, hit.getId(), new Text(hit.getType()), documentFields, metaFields); + SearchHit searchHit = new SearchHit(currentId, hit.getId(), documentFields, metaFields); searchHit.sourceRef(hit.getSourceRef()); searchHit.getSourceAsMap().clear(); Map sourceAsMap = hit.getSourceAsMap(); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/AggregationQueryAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/AggregationQueryAction.java index fd36260fe9..24194e8de5 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/AggregationQueryAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/AggregationQueryAction.java @@ -467,11 +467,6 @@ private void setWhere(Where where) throws SqlParseException { */ private void setIndicesAndTypes() { request.setIndices(query.getIndexArr()); - - String[] typeArr = query.getTypeArr(); - if (typeArr != null) { - request.setTypes(typeArr); - } } private void setLimitFromHint(List hints) { diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/DefaultQueryAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/DefaultQueryAction.java index 6d08164e70..0ed5043ac8 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/DefaultQueryAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/DefaultQueryAction.java @@ -130,11 +130,6 @@ public Select getSelect() { */ private void setIndicesAndTypes() { request.setIndices(query.getIndexArr()); - - String[] typeArr = query.getTypeArr(); - if (typeArr != null) { - request.setTypes(typeArr); - } } /** diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/DeleteQueryAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/DeleteQueryAction.java index f7dd3ecfe7..892c5aeb2d 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/DeleteQueryAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/DeleteQueryAction.java @@ -47,14 +47,6 @@ private void setIndicesAndTypes() { DeleteByQueryRequest innerRequest = request.request(); innerRequest.indices(query.getIndexArr()); - String[] typeArr = query.getTypeArr(); - if (typeArr != null) { - innerRequest.getSearchRequest().types(typeArr); - } -// String[] typeArr = query.getTypeArr(); -// if (typeArr != null) { -// request.set(typeArr); -// } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/maker/Maker.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/maker/Maker.java index a3040c9f44..951d87a2c8 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/maker/Maker.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/maker/Maker.java @@ -307,14 +307,13 @@ private ToXContent make(Condition cond, String name, Object value) throws SqlPar case IDS_QUERY: Object[] idsParameters = (Object[]) value; String[] ids; - String type = idsParameters[0].toString(); if (idsParameters.length == 2 && idsParameters[1] instanceof SubQueryExpression) { Object[] idsFromSubQuery = ((SubQueryExpression) idsParameters[1]).getValues(); ids = arrayOfObjectsToStringArray(idsFromSubQuery, 0, idsFromSubQuery.length - 1); } else { ids = arrayOfObjectsToStringArray(idsParameters, 1, idsParameters.length - 1); } - toXContent = QueryBuilders.idsQuery(type).addIds(ids); + toXContent = QueryBuilders.idsQuery().addIds(ids); break; case NESTED_COMPLEX: case NOT_EXISTS_NESTED_COMPLEX: diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRow.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRow.java index 54fa318f39..8f418deadb 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRow.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRow.java @@ -149,9 +149,6 @@ private SearchHit cloneHit(Row other) { SearchHit combined = new SearchHit( hit.docId(), hit.getId() + "|" + (other == NULL ? "0" : ((SearchHitRow) other).hit.getId()), - new Text( - hit.getType() + "|" + (other == NULL ? null : ((SearchHitRow) other).hit.getType()) - ), documentFields, metaFields ); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/request/SqlRequest.java b/legacy/src/main/java/org/opensearch/sql/legacy/request/SqlRequest.java index fc31515c09..14acb26b3c 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/request/SqlRequest.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/request/SqlRequest.java @@ -89,7 +89,7 @@ private boolean hasFilterInRequest() { private void addFilterFromJson(BoolQueryBuilder boolQuery) throws SqlParseException { try { String filter = getFilterObjectAsString(jsonContent); - SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList()); + SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); XContentParser parser = XContentFactory.xContent(XContentType.JSON). createParser(new NamedXContentRegistry(searchModule.getNamedXContents()), LoggingDeprecationHandler.INSTANCE, diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/join/JoinRewriteRule.java b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/join/JoinRewriteRule.java index 7c24d6c8db..b32803561e 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/join/JoinRewriteRule.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/join/JoinRewriteRule.java @@ -107,8 +107,8 @@ public void rewrite(SQLQueryExpr root) { tableNameToAlias.put(table.getName(), table.getAlias()); - FieldMappings fieldMappings = clusterState.getFieldMappings( - new String[]{tableName}).firstMapping().firstMapping(); + FieldMappings fieldMappings = clusterState. getFieldMappings( + new String[]{tableName}).firstMapping(); fieldMappings.flat((fieldName, type) -> tableByFieldName.put(fieldName, table)); }); diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/matchtoterm/TermFieldRewriter.java b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/matchtoterm/TermFieldRewriter.java index 87d9541045..5890befbca 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/matchtoterm/TermFieldRewriter.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/matchtoterm/TermFieldRewriter.java @@ -125,7 +125,7 @@ public boolean visit(SQLIdentifierExpr expr) { String fullFieldName = arr[1]; String index = curScope().getAliases().get(alias); - FieldMappings fieldMappings = curScope().getMapper().mapping(index).firstMapping(); + FieldMappings fieldMappings = curScope().getMapper().mapping(index); if (fieldMappings.has(fullFieldName)) { source = fieldMappings.mapping(fullFieldName); } else { diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/matchtoterm/TermFieldScope.java b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/matchtoterm/TermFieldScope.java index e8206cf82b..fd6380d9d1 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/matchtoterm/TermFieldScope.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/matchtoterm/TermFieldScope.java @@ -6,11 +6,7 @@ package org.opensearch.sql.legacy.rewriter.matchtoterm; -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; -import java.util.Optional; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import org.json.JSONObject; import org.opensearch.sql.legacy.esdomain.mapping.FieldMappings; @@ -48,9 +44,7 @@ public void setMapper(IndexMappings mapper) { } public Optional> resolveFieldMapping(String fieldName) { - Set indexMappings = mapper.allMappings().stream(). - flatMap(typeMappings -> typeMappings.allMappings().stream()). - collect(Collectors.toSet()); + Set indexMappings = new HashSet<>(mapper.allMappings()); Optional> resolvedMapping = indexMappings.stream() .filter(mapping -> mapping.has(fieldName)) diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/esdomain/mapping/FieldMappingsTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/esdomain/mapping/FieldMappingsTest.java index 32af16bfe7..412c351c56 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/esdomain/mapping/FieldMappingsTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/esdomain/mapping/FieldMappingsTest.java @@ -45,7 +45,7 @@ public void cleanUp() { @Test public void flatFieldMappingsShouldIncludeFieldsOnAllLevels() { IndexMappings indexMappings = LocalClusterState.state().getFieldMappings(new String[]{"field_mappings"}); - FieldMappings fieldMappings = indexMappings.firstMapping().firstMapping(); + FieldMappings fieldMappings = indexMappings.firstMapping(); Map typeByFieldName = new HashMap<>(); fieldMappings.flat(typeByFieldName::put); diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/LocalClusterStateTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/LocalClusterStateTest.java index 6bc7f22262..67ad4b9ba2 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/LocalClusterStateTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/LocalClusterStateTest.java @@ -33,7 +33,6 @@ import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.esdomain.mapping.FieldMappings; import org.opensearch.sql.legacy.esdomain.mapping.IndexMappings; -import org.opensearch.sql.legacy.esdomain.mapping.TypeMappings; import org.opensearch.sql.legacy.util.TestsConstants; import org.opensearch.sql.opensearch.setting.OpenSearchSettings; @@ -126,10 +125,7 @@ public void getMappingForExistingField() { IndexMappings indexMappings = LocalClusterState.state().getFieldMappings(new String[]{INDEX_NAME}); Assert.assertNotNull(indexMappings); - TypeMappings typeMappings = indexMappings.mapping(INDEX_NAME); - Assert.assertNotNull(typeMappings); - - FieldMappings fieldMappings = typeMappings.mapping(TYPE_NAME); + FieldMappings fieldMappings = indexMappings.mapping(INDEX_NAME); Assert.assertNotNull(fieldMappings); Assert.assertEquals("text", fieldMappings.mapping("address").get("type")); @@ -144,8 +140,7 @@ public void getMappingForExistingField() { @Test public void getMappingForInvalidField() { IndexMappings indexMappings = LocalClusterState.state().getFieldMappings(new String[]{INDEX_NAME}); - TypeMappings typeMappings = indexMappings.mapping(INDEX_NAME); - FieldMappings fieldMappings = typeMappings.mapping(TYPE_NAME); + FieldMappings fieldMappings = indexMappings.mapping(INDEX_NAME); Assert.assertNull(fieldMappings.mapping("work-email")); Assert.assertNull(fieldMappings.mapping("manager.home-address")); @@ -168,7 +163,7 @@ public void getMappingFromCache() throws IOException { for (int i = 0; i < 10; i++) { LocalClusterState.state().getFieldMappings(new String[]{INDEX_NAME}); } - verify(mockService.state().metadata(), times(1)).findMappings(eq(new String[]{INDEX_NAME}), any(), any()); + verify(mockService.state().metadata(), times(1)).findMappings(eq(new String[]{INDEX_NAME}), any()); // 2.Fire cluster state change event Assert.assertNotNull(listener[0]); @@ -180,7 +175,7 @@ public void getMappingFromCache() throws IOException { for (int i = 0; i < 5; i++) { LocalClusterState.state().getFieldMappings(new String[]{INDEX_NAME}); } - verify(mockService.state().metadata(), times(2)).findMappings(eq(new String[]{INDEX_NAME}), any(), any()); + verify(mockService.state().metadata(), times(2)).findMappings(eq(new String[]{INDEX_NAME}), any()); } @Test diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/executor/format/CSVResultsExtractorTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/executor/format/CSVResultsExtractorTest.java index 1666112fac..b3afff2ce1 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/executor/format/CSVResultsExtractorTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/executor/format/CSVResultsExtractorTest.java @@ -19,7 +19,7 @@ import org.opensearch.sql.legacy.expression.domain.BindingTuple; public class CSVResultsExtractorTest { - private final CSVResultsExtractor csvResultsExtractor = new CSVResultsExtractor(false, false, false); + private final CSVResultsExtractor csvResultsExtractor = new CSVResultsExtractor(false, false); @Test public void extractResultsFromBindingTupleListShouldPass() throws CsvExtractorException { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/util/CheckScriptContents.java b/legacy/src/test/java/org/opensearch/sql/legacy/util/CheckScriptContents.java index a14918c0d1..f427170cfc 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/util/CheckScriptContents.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/util/CheckScriptContents.java @@ -119,7 +119,6 @@ public static boolean scriptHasPattern(ScriptFilter scriptFilter, String regex) } public static void stubMockClient(Client mockClient) { - try { String mappings = "{\n" + " \"opensearch-sql_test_index_bank\": {\n" + " \"mappings\": {\n" + @@ -203,15 +202,7 @@ public static void stubMockClient(Client mockClient) { ActionFuture mockActionResp = mock(ActionFuture.class); when(mockIndexClient.getFieldMappings(any(GetFieldMappingsRequest.class))).thenReturn(mockActionResp); - - when(mockActionResp.actionGet()).thenReturn(GetFieldMappingsResponse.fromXContent(createParser(mappings))); - mockLocalClusterState(mappings); - - } catch (IOException e) { - throw new ParserException(e.getMessage()); - } - } public static XContentParser createParser(String mappings) throws IOException { @@ -236,9 +227,9 @@ public static ClusterService mockClusterService(String mappings) { when(mockService.state()).thenReturn(mockState); when(mockState.metadata()).thenReturn(mockMetaData); try { - ImmutableOpenMap.Builder> builder = ImmutableOpenMap.builder(); - builder.put(TestsConstants.TEST_INDEX_BANK, IndexMetadata.fromXContent(createParser(mappings)).getMappings()); - when(mockMetaData.findMappings(any(), any(), any())).thenReturn(builder.build()); + ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); + builder.put(TestsConstants.TEST_INDEX_BANK, IndexMetadata.fromXContent(createParser(mappings)).mapping()); + when(mockMetaData.findMappings(any(), any())).thenReturn(builder.build()); } catch (IOException e) { throw new IllegalStateException(e); diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/util/MultipleIndexClusterUtils.java b/legacy/src/test/java/org/opensearch/sql/legacy/util/MultipleIndexClusterUtils.java index 6bc90f111d..0213cac01b 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/util/MultipleIndexClusterUtils.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/util/MultipleIndexClusterUtils.java @@ -139,7 +139,7 @@ public class MultipleIndexClusterUtils { "}"; public static void mockMultipleIndexEnv() { - mockLocalClusterState(new ImmutableMap.Builder>>() + mockLocalClusterState(new ImmutableMap.Builder>() .put(INDEX_ACCOUNT_1, buildIndexMapping(INDEX_ACCOUNT_1, INDEX_ACCOUNT_1_MAPPING)) .put(INDEX_ACCOUNT_2, buildIndexMapping(INDEX_ACCOUNT_2, INDEX_ACCOUNT_2_MAPPING)) .put(INDEX_ACCOUNT_ALL, buildIndexMapping(new ImmutableMap.Builder() @@ -149,14 +149,14 @@ public static void mockMultipleIndexEnv() { .build()); } - public static void mockLocalClusterState(Map>> indexMapping) { + public static void mockLocalClusterState(Map> indexMapping) { LocalClusterState.state().setClusterService(mockClusterService(indexMapping)); LocalClusterState.state().setResolver(mockIndexNameExpressionResolver()); LocalClusterState.state().setPluginSettings(mockPluginSettings()); } - public static ClusterService mockClusterService(Map>> indexMapping) { + public static ClusterService mockClusterService(Map> indexMapping) { ClusterService mockService = mock(ClusterService.class); ClusterState mockState = mock(ClusterState.class); Metadata mockMetaData = mock(Metadata.class); @@ -164,8 +164,8 @@ public static ClusterService mockClusterService(Map>> entry : indexMapping.entrySet()) { - when(mockMetaData.findMappings(eq(new String[]{entry.getKey()}), any(), any())).thenReturn(entry.getValue()); + for (Map.Entry> entry : indexMapping.entrySet()) { + when(mockMetaData.findMappings(eq(new String[]{entry.getKey()}), any())).thenReturn(entry.getValue()); } } catch (IOException e) { throw new IllegalStateException(e); @@ -173,11 +173,11 @@ public static ClusterService mockClusterService(Map> buildIndexMapping(Map indexMapping) { + private static ImmutableOpenMap buildIndexMapping(Map indexMapping) { try { - ImmutableOpenMap.Builder> builder = ImmutableOpenMap.builder(); + ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); for (Map.Entry entry : indexMapping.entrySet()) { - builder.put(entry.getKey(), IndexMetadata.fromXContent(createParser(entry.getValue())).getMappings()); + builder.put(entry.getKey(), IndexMetadata.fromXContent(createParser(entry.getValue())).mapping()); } return builder.build(); } catch (IOException e) { @@ -185,11 +185,10 @@ private static ImmutableOpenMap> buildIndexMapping(String index, - String mapping) { + private static ImmutableOpenMap buildIndexMapping(String index, String mapping) { try { - ImmutableOpenMap.Builder> builder = ImmutableOpenMap.builder(); - builder.put(index, IndexMetadata.fromXContent(createParser(mapping)).getMappings()); + ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); + builder.put(index, IndexMetadata.fromXContent(createParser(mapping)).mapping()); return builder.build(); } catch (IOException e) { throw new IllegalStateException(e); diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/util/TestUtils.java b/legacy/src/test/java/org/opensearch/sql/legacy/util/TestUtils.java index 67a48d50a6..27be512fc0 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/util/TestUtils.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/util/TestUtils.java @@ -698,8 +698,6 @@ public static void loadBulk(Client client, String jsonPath, String defaultIndex) IndexRequest indexRequest = new IndexRequest(); indexRequest.index(defaultIndex); - String docType = actionJson.getJSONObject("index").getString("_type"); - indexRequest.type(docType); if (actionJson.getJSONObject("index").has("_id")) { String docId = actionJson.getJSONObject("index").getString("_id"); indexRequest.id(docId); diff --git a/opensearch/build.gradle b/opensearch/build.gradle index eed277d938..7f8c853b1a 100644 --- a/opensearch/build.gradle +++ b/opensearch/build.gradle @@ -23,28 +23,28 @@ */ plugins { - id 'java' + id 'java-library' id "io.freefair.lombok" id 'jacoco' } dependencies { - compile project(':core') - compile group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" - compile "io.github.resilience4j:resilience4j-retry:1.5.0" - compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.12.6' - compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.12.6' - compile group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: '2.12.6' - compile group: 'org.json', name: 'json', version:'20180813' + api project(':core') + api group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" + implementation "io.github.resilience4j:resilience4j-retry:1.5.0" + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.12.6' + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.12.6' + implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: '2.12.6' + implementation group: 'org.json', name: 'json', version:'20180813' compileOnly group: 'org.opensearch.client', name: 'opensearch-rest-high-level-client', version: "${opensearch_version}" - compile group: 'org.opensearch', name:'opensearch-ml-client', version: '1.3.0.0-SNAPSHOT' + implementation group: 'org.opensearch', name:'opensearch-ml-client', version: '1.3.0.0-SNAPSHOT' testImplementation('org.junit.jupiter:junit-jupiter:5.6.2') - testCompile group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' - testCompile group: 'org.mockito', name: 'mockito-core', version: '3.5.0' - testCompile group: 'org.mockito', name: 'mockito-junit-jupiter', version: '3.5.0' - testCompile group: 'org.opensearch.client', name: 'opensearch-rest-high-level-client', version: "${opensearch_version}" - testCompile group: 'org.opensearch.test', name: 'framework', version: "${opensearch_version}" + testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' + testImplementation group: 'org.mockito', name: 'mockito-core', version: '3.5.0' + testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: '3.5.0' + testImplementation group: 'org.opensearch.client', name: 'opensearch-rest-high-level-client', version: "${opensearch_version}" + testImplementation group: 'org.opensearch.test', name: 'framework', version: "${opensearch_version}" } test { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index b66a1dc7ed..fe26280812 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -12,7 +12,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -38,9 +37,6 @@ /** OpenSearch connection by node client. */ public class OpenSearchNodeClient implements OpenSearchClient { - /** Default types and field filter to match all. */ - public static final String[] ALL_TYPES = new String[0]; - public static final Function> ALL_FIELDS = (anyIndex -> (anyField -> true)); @@ -83,7 +79,7 @@ public Map getIndexMappings(String... indexExpression) { String[] concreteIndices = resolveIndexExpression(state, indexExpression); return populateIndexMappings( - state.metadata().findMappings(concreteIndices, ALL_TYPES, ALL_FIELDS)); + state.metadata().findMappings(concreteIndices, ALL_FIELDS)); } catch (IOException e) { throw new IllegalStateException( "Failed to read mapping in cluster state for index pattern [" + indexExpression + "]", e); @@ -157,24 +153,16 @@ private String[] resolveIndexExpression(ClusterState state, String[] indices) { } private Map populateIndexMappings( - ImmutableOpenMap> indexMappings) { + ImmutableOpenMap indexMappings) { ImmutableMap.Builder result = ImmutableMap.builder(); - for (ObjectObjectCursor> cursor : - indexMappings) { - result.put(cursor.key, populateIndexMapping(cursor.value)); + for (ObjectObjectCursor cursor: + indexMappings) { + result.put(cursor.key, new IndexMapping(cursor.value)); } return result.build(); } - private IndexMapping populateIndexMapping( - ImmutableOpenMap indexMapping) { - if (indexMapping.isEmpty()) { - return new IndexMapping(Collections.emptyMap()); - } - return new IndexMapping(indexMapping.iterator().next().value); - } - /** Copy from LogUtils. */ private static Runnable withCurrentContext(final Runnable task) { final Map currentContext = ThreadContext.getImmutableContext(); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java index bcb318793c..50410e07cc 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java @@ -313,16 +313,18 @@ public ClusterService mockClusterService(String indexName, String mappings) { when(mockService.state()).thenReturn(mockState); when(mockState.metadata()).thenReturn(mockMetaData); try { - ImmutableOpenMap.Builder> builder = + ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); - ImmutableOpenMap metadata; + MappingMetadata metadata; if (mappings.isEmpty()) { - metadata = ImmutableOpenMap.of(); + metadata = MappingMetadata.EMPTY_MAPPINGS; } else { - metadata = IndexMetadata.fromXContent(createParser(mappings)).getMappings(); + metadata = IndexMetadata.fromXContent(createParser(mappings)).mapping(); } + + builder.put(indexName, metadata); - when(mockMetaData.findMappings(any(), any(), any())).thenReturn(builder.build()); + when(mockMetaData.findMappings(any(), any())).thenReturn(builder.build()); // IndexNameExpressionResolver use this method to check if index exists. If not, // IndexNotFoundException is thrown. @@ -342,7 +344,7 @@ public ClusterService mockClusterService(String indexName, Throwable t) { when(mockService.state()).thenReturn(mockState); when(mockState.metadata()).thenReturn(mockMetaData); try { - when(mockMetaData.findMappings(any(), any(), any())).thenThrow(t); + when(mockMetaData.findMappings(any(), any())).thenThrow(t); when(mockMetaData.getIndicesLookup()) .thenReturn(ImmutableSortedMap.of(indexName, mock(IndexAbstraction.class))); } catch (IOException e) { diff --git a/plugin/build.gradle b/plugin/build.gradle index b934a804a3..7fa24f62a5 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -65,10 +65,10 @@ configurations.all { } dependencies { - compile group: 'org.springframework', name: 'spring-beans', version: '5.2.19.RELEASE' - compile project(":ppl") - compile project(':legacy') - compile project(':opensearch') + api group: 'org.springframework', name: 'spring-beans', version: '5.2.19.RELEASE' + api project(":ppl") + api project(':legacy') + api project(':opensearch') } diff --git a/ppl/build.gradle b/ppl/build.gradle index 4b88c000fe..e59f89f91c 100644 --- a/ppl/build.gradle +++ b/ppl/build.gradle @@ -23,7 +23,7 @@ */ plugins { - id 'java' + id 'java-library' id "io.freefair.lombok" id 'jacoco' id 'antlr' @@ -44,20 +44,20 @@ configurations { dependencies { antlr "org.antlr:antlr4:4.7.1" - compile "org.antlr:antlr4-runtime:4.7.1" - compile group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' - compile group: 'org.opensearch', name: 'opensearch-x-content', version: "${opensearch_version}" - compile group: 'org.json', name: 'json', version: '20180813' - compile group: 'org.springframework', name: 'spring-context', version: '5.2.19.RELEASE' - compile group: 'org.springframework', name: 'spring-beans', version: '5.2.19.RELEASE' - compile group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.17.1' - compile project(':common') - compile project(':core') - compile project(':protocol') + implementation "org.antlr:antlr4-runtime:4.7.1" + implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' + api group: 'org.opensearch', name: 'opensearch-x-content', version: "${opensearch_version}" + api group: 'org.json', name: 'json', version: '20180813' + implementation group: 'org.springframework', name: 'spring-context', version: '5.2.19.RELEASE' + implementation group: 'org.springframework', name: 'spring-beans', version: '5.2.19.RELEASE' + implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.17.1' + api project(':common') + api project(':core') + api project(':protocol') - testCompile group: 'junit', name: 'junit', version: '4.13.2' - testCompile group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' - testCompile group: 'org.mockito', name: 'mockito-core', version: '3.3.3' + testImplementation group: 'junit', name: 'junit', version: '4.13.2' + testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' + testImplementation group: 'org.mockito', name: 'mockito-core', version: '3.3.3' } diff --git a/protocol/build.gradle b/protocol/build.gradle index 5e1464a0a4..9af3dd733c 100644 --- a/protocol/build.gradle +++ b/protocol/build.gradle @@ -29,18 +29,18 @@ plugins { } dependencies { - compile group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' - compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.12.6' - compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.12.6' - compile group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: '2.12.6' + implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.12.6' + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.12.6' + implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: '2.12.6' implementation 'com.google.code.gson:gson:2.8.9' - compile project(':core') - compile project(':opensearch') + implementation project(':core') + implementation project(':opensearch') testImplementation('org.junit.jupiter:junit-jupiter:5.6.2') - testCompile group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' - testCompile group: 'org.mockito', name: 'mockito-core', version: '3.3.3' - testCompile group: 'org.mockito', name: 'mockito-junit-jupiter', version: '3.3.3' + testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' + testImplementation group: 'org.mockito', name: 'mockito-core', version: '3.3.3' + testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: '3.3.3' } test { diff --git a/sql-jdbc/build.gradle b/sql-jdbc/build.gradle index b1f45b0994..9042968e7c 100644 --- a/sql-jdbc/build.gradle +++ b/sql-jdbc/build.gradle @@ -24,7 +24,7 @@ plugins { group 'org.opensearch.client' // keep version in sync with version in Driver source -version '1.3.0.0' +version '2.0.0.0' boolean snapshot = "true".equals(System.getProperty("build.snapshot", "false")); if (snapshot) { diff --git a/sql-odbc/src/CMakeLists.txt b/sql-odbc/src/CMakeLists.txt index 7e940ca63e..7ba1d57e08 100644 --- a/sql-odbc/src/CMakeLists.txt +++ b/sql-odbc/src/CMakeLists.txt @@ -63,8 +63,8 @@ set(INSTALL_SRC "${CMAKE_CURRENT_SOURCE_DIR}/installer") set(DSN_INSTALLER_SRC "${CMAKE_CURRENT_SOURCE_DIR}/DSNInstaller") # ODBC Driver version -set(DRIVER_PACKAGE_VERSION "1.3.0.0") -set(DRIVER_PACKAGE_VERSION_COMMA_SEPARATED "1,3,0,0") +set(DRIVER_PACKAGE_VERSION "2.0.0.0") +set(DRIVER_PACKAGE_VERSION_COMMA_SEPARATED "2,0,0,0") add_compile_definitions( OPENSEARCH_ODBC_VERSION="${DRIVER_PACKAGE_VERSION}" # Comma separated version is required for odbc administrator's driver file. OPENSEARCH_ODBC_DRVFILE_VERSION=${DRIVER_PACKAGE_VERSION_COMMA_SEPARATED} ) diff --git a/sql/build.gradle b/sql/build.gradle index 1acb115137..b851760910 100644 --- a/sql/build.gradle +++ b/sql/build.gradle @@ -44,19 +44,19 @@ configurations { dependencies { antlr "org.antlr:antlr4:4.7.1" - compile "org.antlr:antlr4-runtime:4.7.1" + implementation "org.antlr:antlr4-runtime:4.7.1" implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' - compile group: 'org.json', name: 'json', version:'20180813' - compile group: 'org.springframework', name: 'spring-context', version: '5.2.19.RELEASE' - compile group: 'org.springframework', name: 'spring-beans', version: '5.2.19.RELEASE' - compile project(':common') - compile project(':core') - compile project(':protocol') + implementation group: 'org.json', name: 'json', version:'20180813' + implementation group: 'org.springframework', name: 'spring-context', version: '5.2.19.RELEASE' + implementation group: 'org.springframework', name: 'spring-beans', version: '5.2.19.RELEASE' + implementation project(':common') + implementation project(':core') + api project(':protocol') testImplementation('org.junit.jupiter:junit-jupiter:5.6.2') - testCompile group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' - testCompile group: 'org.mockito', name: 'mockito-core', version: '3.3.3' - testCompile group: 'org.mockito', name: 'mockito-junit-jupiter', version: '3.3.3' + testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' + testImplementation group: 'org.mockito', name: 'mockito-core', version: '3.3.3' + testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: '3.3.3' } test { diff --git a/workbench/opensearch_dashboards.json b/workbench/opensearch_dashboards.json index 339e6c6c56..11d4c9c070 100644 --- a/workbench/opensearch_dashboards.json +++ b/workbench/opensearch_dashboards.json @@ -1,7 +1,7 @@ { "id": "queryWorkbenchDashboards", - "version": "1.3.0.0", - "opensearchDashboardsVersion": "1.3.0", + "version": "2.0.0.0", + "opensearchDashboardsVersion": "2.0.0", "server": true, "ui": true, "requiredPlugins": ["navigation"], diff --git a/workbench/package.json b/workbench/package.json index d53daaabb3..cd709d6779 100644 --- a/workbench/package.json +++ b/workbench/package.json @@ -1,12 +1,12 @@ { "name": "opensearch-query-workbench", - "version": "1.3.0.0", + "version": "2.0.0.0", "description": "Query Workbench", "main": "index.js", "license": "Apache-2.0", "homepage": "https://github.com/opensearch-project/sql/tree/main/workbench", "opensearchDashboards": { - "version": "1.3.0", + "version": "2.0.0", "templateVersion": "1.0.0" }, "repository": { @@ -31,7 +31,7 @@ "devDependencies": { "@testing-library/user-event": "^13.1.9", "@types/hapi-latest": "npm:@types/hapi@18.0.3", - "@types/react-router-dom": "^5.1.5", + "@types/react-router-dom": "^5.3.2", "cypress": "^5.0.0", "eslint": "^6.8.0", "eslint-plugin-no-unsanitized": "^3.0.2", @@ -47,7 +47,7 @@ "tslint-plugin-prettier": "^2.0.1" }, "engines": { - "node": "10.24.1", + "node": "14.18.2", "yarn": "^1.21.1" }, "resolutions": { diff --git a/workbench/public/components/QueryResults/__snapshots__/QueryResults.test.tsx.snap b/workbench/public/components/QueryResults/__snapshots__/QueryResults.test.tsx.snap index f625b404e8..9411b874ea 100644 --- a/workbench/public/components/QueryResults/__snapshots__/QueryResults.test.tsx.snap +++ b/workbench/public/components/QueryResults/__snapshots__/QueryResults.test.tsx.snap @@ -764,7 +764,7 @@ exports[` spec renders the component to test tabs down >
spec renders the component to test tabs down - Eddie + Teddy
@@ -840,7 +840,7 @@ exports[` spec renders the component to test tabs down - Eddie Underwood + Teddy Underwood @@ -929,7 +929,7 @@ exports[` spec renders the component to test tabs down - eddie@underwood-family.zzz + Teddy@underwood-family.zzz @@ -1099,7 +1099,7 @@ exports[` spec renders the component to test tabs down - eddie + teddy @@ -1132,7 +1132,7 @@ exports[` spec renders the component to test tabs down >
spec renders the component to test tabs down - Teddy + Eddie
@@ -1208,7 +1208,7 @@ exports[` spec renders the component to test tabs down - Teddy Underwood + Eddie Underwood @@ -1297,7 +1297,7 @@ exports[` spec renders the component to test tabs down - Teddy@underwood-family.zzz + eddie@underwood-family.zzz @@ -1467,7 +1467,7 @@ exports[` spec renders the component to test tabs down - teddy + eddie @@ -2972,7 +2972,7 @@ exports[` spec renders the component to test tabs down >
spec renders the component to test tabs down >
spec renders the component to test tabs down - Eddie + Teddy
@@ -5302,7 +5302,7 @@ exports[` spec renders the component to test tabs down - Eddie Underwood + Teddy Underwood
@@ -5391,7 +5391,7 @@ exports[` spec renders the component to test tabs down - eddie@underwood-family.zzz + Teddy@underwood-family.zzz @@ -5561,7 +5561,7 @@ exports[` spec renders the component to test tabs down - eddie + teddy @@ -5594,7 +5594,7 @@ exports[` spec renders the component to test tabs down >
spec renders the component to test tabs down - Teddy + Eddie
@@ -5670,7 +5670,7 @@ exports[` spec renders the component to test tabs down - Teddy Underwood + Eddie Underwood @@ -5759,7 +5759,7 @@ exports[` spec renders the component to test tabs down - Teddy@underwood-family.zzz + eddie@underwood-family.zzz @@ -5929,7 +5929,7 @@ exports[` spec renders the component to test tabs down - teddy + eddie @@ -7434,7 +7434,7 @@ exports[` spec renders the component to test tabs down >
spec renders the component with mock query re >
spec renders the component with mock query re - Eddie + Teddy
@@ -9857,7 +9857,7 @@ exports[` spec renders the component with mock query re - Eddie Underwood + Teddy Underwood
@@ -9946,7 +9946,7 @@ exports[` spec renders the component with mock query re - eddie@underwood-family.zzz + Teddy@underwood-family.zzz @@ -10116,7 +10116,7 @@ exports[` spec renders the component with mock query re - eddie + teddy @@ -10149,7 +10149,7 @@ exports[` spec renders the component with mock query re >
spec renders the component with mock query re - Teddy + Eddie
@@ -10225,7 +10225,7 @@ exports[` spec renders the component with mock query re - Teddy Underwood + Eddie Underwood @@ -10314,7 +10314,7 @@ exports[` spec renders the component with mock query re - Teddy@underwood-family.zzz + eddie@underwood-family.zzz @@ -10484,7 +10484,7 @@ exports[` spec renders the component with mock query re - teddy + eddie @@ -11989,7 +11989,7 @@ exports[` spec renders the component with mock query re >
spec renders the component with mock query re >
spec renders the component with mock query re - Eddie + Teddy
@@ -14319,7 +14319,7 @@ exports[` spec renders the component with mock query re - Eddie Underwood + Teddy Underwood
@@ -14408,7 +14408,7 @@ exports[` spec renders the component with mock query re - eddie@underwood-family.zzz + Teddy@underwood-family.zzz @@ -14578,7 +14578,7 @@ exports[` spec renders the component with mock query re - eddie + teddy @@ -14611,7 +14611,7 @@ exports[` spec renders the component with mock query re >
spec renders the component with mock query re - Teddy + Eddie
@@ -14687,7 +14687,7 @@ exports[` spec renders the component with mock query re - Teddy Underwood + Eddie Underwood @@ -14776,7 +14776,7 @@ exports[` spec renders the component with mock query re - Teddy@underwood-family.zzz + eddie@underwood-family.zzz @@ -14946,7 +14946,7 @@ exports[` spec renders the component with mock query re - teddy + eddie @@ -16451,7 +16451,7 @@ exports[` spec renders the component with mock query re >
spec renders component with mock QueryResults data >
spec renders component with mock QueryResults data - Eddie + Teddy
@@ -616,7 +616,7 @@ exports[` spec renders component with mock QueryResults data - Eddie Underwood + Teddy Underwood
@@ -705,7 +705,7 @@ exports[` spec renders component with mock QueryResults data - eddie@underwood-family.zzz + Teddy@underwood-family.zzz @@ -875,7 +875,7 @@ exports[` spec renders component with mock QueryResults data - eddie + teddy @@ -908,7 +908,7 @@ exports[` spec renders component with mock QueryResults data >
spec renders component with mock QueryResults data - Teddy + Eddie
@@ -984,7 +984,7 @@ exports[` spec renders component with mock QueryResults data - Teddy Underwood + Eddie Underwood @@ -1073,7 +1073,7 @@ exports[` spec renders component with mock QueryResults data - Teddy@underwood-family.zzz + eddie@underwood-family.zzz @@ -1243,7 +1243,7 @@ exports[` spec renders component with mock QueryResults data - teddy + eddie @@ -2748,7 +2748,7 @@ exports[` spec renders component with mock QueryResults data >
spec renders component with mock QueryResults data >
spec renders component with mock QueryResults data - Eddie + Teddy
@@ -5285,7 +5285,7 @@ exports[` spec renders component with mock QueryResults data - Eddie Underwood + Teddy Underwood
@@ -5374,7 +5374,7 @@ exports[` spec renders component with mock QueryResults data - eddie@underwood-family.zzz + Teddy@underwood-family.zzz @@ -5544,7 +5544,7 @@ exports[` spec renders component with mock QueryResults data - eddie + teddy @@ -5576,7 +5576,7 @@ exports[` spec renders component with mock QueryResults data class="euiTableRow" >
@@ -5999,7 +5999,7 @@ exports[` spec renders component with mock QueryResults data - Teddy@underwood-family.zzz + eddie@underwood-family.zzz @@ -6169,7 +6169,7 @@ exports[` spec renders component with mock QueryResults data - teddy + eddie @@ -7674,7 +7674,7 @@ exports[` spec renders component with mock QueryResults data >
spec renders component with mock QueryResults data >
spec renders component with mock QueryResults data - Eddie + Teddy
@@ -10108,7 +10108,7 @@ exports[` spec renders component with mock QueryResults data - Eddie Underwood + Teddy Underwood
@@ -10197,7 +10197,7 @@ exports[` spec renders component with mock QueryResults data - eddie@underwood-family.zzz + Teddy@underwood-family.zzz @@ -10367,7 +10367,7 @@ exports[` spec renders component with mock QueryResults data - eddie + teddy @@ -10400,7 +10400,7 @@ exports[` spec renders component with mock QueryResults data >
spec renders component with mock QueryResults data - Teddy + Eddie
@@ -10476,7 +10476,7 @@ exports[` spec renders component with mock QueryResults data - Teddy Underwood + Eddie Underwood @@ -10565,7 +10565,7 @@ exports[` spec renders component with mock QueryResults data - Teddy@underwood-family.zzz + eddie@underwood-family.zzz @@ -10735,7 +10735,7 @@ exports[` spec renders component with mock QueryResults data - teddy + eddie @@ -12240,7 +12240,7 @@ exports[` spec renders component with mock QueryResults data >
/build/", "/node_modules/"], + testEnvironment: 'jsdom', }; diff --git a/workbench/yarn.lock b/workbench/yarn.lock index 07ac016f23..6707d461a6 100644 --- a/workbench/yarn.lock +++ b/workbench/yarn.lock @@ -142,6 +142,11 @@ resolved "https://registry.yarnpkg.com/@types/history/-/history-4.7.8.tgz#49348387983075705fe8f4e02fb67f7daaec4934" integrity sha512-S78QIYirQcUoo6UJZx9CSP0O2ix9IaeAXwQi26Rhr/+mg7qqPy8TzaxHSUut7eGjL8WmLccT7/MXf304WjqHcA== +"@types/history@^4.7.11": + version "4.7.11" + resolved "https://registry.yarnpkg.com/@types/history/-/history-4.7.11.tgz#56588b17ae8f50c53983a524fc3cc47437969d64" + integrity sha512-qjDJRrmvBMiTx+jyLxvLfJU7UznFuokDv4f3WRuriHKERccVpFU+8XMQUAbDzoiJCsmexxRExQeMwwCdamSKDA== + "@types/iron@*": version "5.0.1" resolved "https://registry.yarnpkg.com/@types/iron/-/iron-5.0.1.tgz#5420bbda8623c48ee51b9a78ebad05d7305b4b24" @@ -231,12 +236,12 @@ "@types/react" "*" "@types/react-router" "*" -"@types/react-router-dom@^5.1.5": - version "5.1.6" - resolved "https://registry.yarnpkg.com/@types/react-router-dom/-/react-router-dom-5.1.6.tgz#07b14e7ab1893a837c8565634960dc398564b1fb" - integrity sha512-gjrxYqxz37zWEdMVvQtWPFMFj1dRDb4TGOcgyOfSXTrEXdF92L00WE3C471O3TV/RF1oskcStkXsOU0Ete4s/g== +"@types/react-router-dom@^5.3.2": + version "5.3.3" + resolved "https://registry.yarnpkg.com/@types/react-router-dom/-/react-router-dom-5.3.3.tgz#e9d6b4a66fcdbd651a5f106c2656a30088cc1e83" + integrity sha512-kpqnYK4wcdm5UaWI3fLcELopqLrHgLqNsdpHauzlQktfkHL3npOSwtj1Uz9oKBAzs7lFtVkV8j83voAz2D8fhw== dependencies: - "@types/history" "*" + "@types/history" "^4.7.11" "@types/react" "*" "@types/react-router" "*" @@ -2297,10 +2302,10 @@ semver-compare@^1.0.0: resolved "https://registry.yarnpkg.com/semver-compare/-/semver-compare-1.0.0.tgz#0dee216a1c941ab37e9efb1788f6afc5ff5537fc" integrity sha1-De4hahyUGrN+nvsXiPavxf9VN/w= -semver-regex@^3.1.3: - version "3.1.3" - resolved "https://registry.yarnpkg.com/semver-regex/-/semver-regex-3.1.3.tgz#b2bcc6f97f63269f286994e297e229b6245d0dc3" - integrity sha512-Aqi54Mk9uYTjVexLnR67rTyBusmwd04cLkHy9hNvk3+G3nT2Oyg7E0l4XVbOaNwIvQ3hHeYxGcyEy+mKreyBFQ== +semver-regex@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/semver-regex/-/semver-regex-2.0.0.tgz#a93c2c5844539a770233379107b38c7b4ac9d338" + integrity sha512-mUdIBBvdn0PLOeP3TEkMH7HHeUP3GjsXCwKarjv/kGmUFOYg1VqEemKhoQpWMu6X2I8kHeuVdGibLGkVK+/5Qw== semver@7.x: version "7.3.4"