diff --git a/.github/workflows/release-workflow.yml b/.github/workflows/release-workflow.yml index 9b4e4ba61f..ff35711e5b 100644 --- a/.github/workflows/release-workflow.yml +++ b/.github/workflows/release-workflow.yml @@ -32,10 +32,10 @@ jobs: - name: Run build run: | - ./gradlew buildPackages --refresh-dependencies --console=plain -Dbuild.snapshot=false - artifact=`ls build/distributions/*.zip` - rpm_artifact=`ls build/distributions/*.rpm` - deb_artifact=`ls build/distributions/*.deb` + ./gradlew build buildDeb buildRpm --no-daemon --refresh-dependencies --console=plain -Dbuild.snapshot=false + artifact=`ls plugin/build/distributions/*.zip` + rpm_artifact=`ls plugin/build/distributions/*.rpm` + deb_artifact=`ls plugin/build/distributions/*.deb` aws s3 cp $artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/elasticsearch-plugins/opendistro-sql/ aws s3 cp $rpm_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/rpms/opendistro-sql/ diff --git a/.github/workflows/test-and-build-workflow.yml b/.github/workflows/test-and-build-workflow.yml index 0c825af4b1..cddcc9a650 100644 --- a/.github/workflows/test-and-build-workflow.yml +++ b/.github/workflows/test-and-build-workflow.yml @@ -21,7 +21,7 @@ jobs: - name: Create Artifact Path run: | mkdir -p opendistro-sql-builds - cp -r ./build/distributions/*.zip opendistro-sql-builds/ + cp -r ./plugin/build/distributions/*.zip opendistro-sql-builds/ - name: Upload Artifacts uses: actions/upload-artifact@v1 diff --git a/.gitignore b/.gitignore index 8ca3d0a2ab..003073b9e4 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,10 @@ gen/ # git mergetool artifact *.orig +gen +*.tokens + +# Python +*/.venv +*/__pycache__ +*.iml diff --git a/build.gradle b/build.gradle index 6cdfd7a3f7..2dcf9726a1 100644 --- a/build.gradle +++ b/build.gradle @@ -15,12 +15,14 @@ buildscript { ext { - es_version = "7.7.0" + es_version = "7.8.0" } - // This isn't applying from repositories.gradle so repeating it here + repositories { mavenCentral() - } + maven { url "https://plugins.gradle.org/m2/" } + jcenter() + } dependencies { classpath "org.elasticsearch.gradle:build-tools:${es_version}" @@ -32,6 +34,7 @@ plugins { id 'java-library' id 'checkstyle' id "io.freefair.lombok" version "5.0.0-rc4" + id 'jacoco' } // Repository on root level is for dependencies that project code depends on. And this block must be placed after plugins{} @@ -40,16 +43,38 @@ repositories { } ext { - opendistroVersion = '1.8.0' + opendistroVersion = '1.9.0' isSnapshot = "true" == System.getProperty("build.snapshot", "true") } -version = "${opendistroVersion}.0" +allprojects { + version = "${opendistroVersion}.0" -apply plugin: 'elasticsearch.esplugin' -apply plugin: 'elasticsearch.testclusters' -apply plugin: 'jacoco' -apply plugin: 'antlr' + plugins.withId('java') { + sourceCompatibility = targetCompatibility = "1.8" + } +} + +// TODO: fix compiler warnings +compileJava.options.warnings = false +compileJava { + options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) + doFirst { + // TODO: do not fail build on warnings, need to fix all compiler warnings + options.compilerArgs.remove('-Werror') + // TODO: need to fix all java doc format + options.compilerArgs.remove('-Xdoclint:all') + } +} +// TODO: Similarly, need to fix compiling errors in test source code +compileTestJava.options.warnings = false +compileTestJava { + options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) + doFirst { + options.compilerArgs.remove('-Werror') + options.compilerArgs.remove('-Xdoclint:all') + } +} jacoco { toolVersion = "0.8.5" @@ -85,246 +110,20 @@ jacocoTestCoverageVerification { } check.dependsOn jacocoTestCoverageVerification -ext { - projectSubstitutions = [:] - licenseFile = rootProject.file('LICENSE.TXT') - noticeFile = rootProject.file('NOTICE') -} - -// ANTLR generated parser file is too large to be checked which caused licenseHeaders stuck. -licenseHeaders { - enabled = true - excludes = ['com/amazon/opendistroforelasticsearch/sql/antlr/parser/**'] -} - -// TODO: need to fix java doc to enable JavaDoc -javadoc.enabled = false -esplugin { - name 'opendistro_sql' // zip file name and plugin name in ${elasticsearch.plugin.name} read by ES when plugin loading - description 'Open Distro for Elasticsearch SQL' - classname 'com.amazon.opendistroforelasticsearch.sql.plugin.SqlPlug' - licenseFile rootProject.file("LICENSE.txt") - noticeFile rootProject.file("NOTICE") -} - -// Set source and target to be compatible with Java 8 -// Commented them out as the default is java 8 -// sourceCompatibility = '1.8' -// targetCompatibility = '1.8' - -// TODO: fix compiler warnings -compileJava.options.warnings = false -compileJava { - options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) - doFirst { - // TODO: do not fail build on warnings, need to fix all compiler warnings - options.compilerArgs.remove('-Werror') - // TODO: need to fix all java doc format - options.compilerArgs.remove('-Xdoclint:all') - } -} - -// TODO: Similarly, need to fix compiling errors in test source code -compileTestJava.options.warnings = false -compileTestJava { - options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) - doFirst { - options.compilerArgs.remove('-Werror') - options.compilerArgs.remove('-Xdoclint:all') - } -} - -// TODO: Need to update integration test to use ElasticSearch test framework -test { - include '**/*Test.class' - exclude 'com/amazon/opendistroforelasticsearch/sql/intgtest/**' - // Gradle runs unit test using a working directory other and project root - // set 'project.root' property to allow unit test classes to access test resources - systemProperty('project.root', project.rootDir.absolutePath) -} - -testClusters.integTest { - testDistribution = 'oss' -} - -integTest.runner { - // add "-Dtests.security.manager=false" to VM options if you want to run integ tests in IntelliJ - systemProperty 'tests.security.manager', 'false' - // allows integration test classes to access test resource from project root path - systemProperty('project.root', project.rootDir.absolutePath) - - systemProperty "https", System.getProperty("https") - systemProperty "user", System.getProperty("user") - systemProperty "password", System.getProperty("password") - - // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for - // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time. - doFirst { systemProperty 'cluster.debug', getDebug()} - - // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable - if (System.getProperty("test.debug") != null) { - jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' - } - - // Run different task based on test type. "exclude" is required for each task. - def testType = System.getProperty("testType") - if (testType == 'doctest') { // Doctest to generate documentation - include 'com/amazon/opendistroforelasticsearch/sql/doctest/**/*IT.class' - exclude 'com/amazon/opendistroforelasticsearch/sql/correctness/**' - exclude 'com/amazon/opendistroforelasticsearch/sql/esintgtest/**' - - } else if (testType == 'comparison') { // Comparision testing - include 'com/amazon/opendistroforelasticsearch/sql/correctness/CorrectnessIT.class' - exclude 'com/amazon/opendistroforelasticsearch/sql/doctest/**' - exclude 'com/amazon/opendistroforelasticsearch/sql/esintgtest/**' - - // Enable logging output to console - testLogging.showStandardStreams true - - // Pass down system properties to IT class - systemProperty "esHost", System.getProperty("esHost") - systemProperty "dbUrl", System.getProperty("dbUrl") - systemProperty "otherDbUrls", System.getProperty("otherDbUrls") - systemProperty "queries", System.getProperty("queries") - - } else { // Run all other integration tests. Skip doctest for now due to randomness in our stats API. - include 'com/amazon/opendistroforelasticsearch/sql/esintgtest/**/*IT.class' - exclude 'com/amazon/opendistroforelasticsearch/sql/doctest/**/*IT.class' - exclude 'com/amazon/opendistroforelasticsearch/sql/correctness/**' - } -} - -generateGrammarSource { - arguments += ['-visitor', '-package', 'com.amazon.opendistroforelasticsearch.sql.antlr.parser'] - source = sourceSets.main.antlr - outputDirectory = file("build/generated-src/antlr/main/com/amazon/opendistroforelasticsearch/sql/antlr/parser") -} - -// Remove ANTLR plugin jars as it's specified as 'compile' dependency internally -configurations { - compile { - extendsFrom = extendsFrom.findAll { it != configurations.antlr } +// TODO: fix code style in main and test source code +subprojects { + apply plugin: 'checkstyle' + checkstyle { + configFile rootProject.file("config/checkstyle/google_checks.xml") + toolVersion "8.20" + configProperties = [ + "org.checkstyle.google.suppressionfilter.config": rootProject.file("config/checkstyle/suppressions.xml")] + ignoreFailures = false } } - -check.dependsOn jacocoTestReport - -// TODO: fix code style in main and test source code checkstyle { configFile file("config/checkstyle/checkstyle.xml") } - checkstyleMain.ignoreFailures = false checkstyleTest.ignoreFailures = true -// TODO: fix forbidden APIs -// from police-man plugin, see https://github.com/policeman-tools/forbidden-apis/wiki/GradleUsage -forbiddenApis.ignoreFailures = true -// TODO: fix forbidden code patterns -// introduced by elasticsearch plugin -// see https://github.com/elastic/elasticsearch/blob/master/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ForbiddenPatternsTask.java -forbiddenPatterns { - setEnabled(false) -} - -// TODO: fix license. skip dependency license checks -dependencyLicenses.enabled = false - -// We don't need to follow ES testing naming conventions. -// see https://github.com/elastic/elasticsearch/blob/323f312bbc829a63056a79ebe45adced5099f6e6/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java -// enable testingConventions check will cause errors like: "Classes ending with [Tests] must subclass [LuceneTestCase]" -testingConventions.enabled = false - -// TODO: need to verify the thirdPartyAudi -// currently it complains missing classes like ibatis, mysql etc, should not be a problem -thirdPartyAudit.enabled = false - - -//****************************************************************************/ -// Dependencies -//****************************************************************************/ -dependencies { - compile group: 'com.alibaba', name: 'druid', version:'1.0.15' - compile group: 'org.locationtech.spatial4j', name: 'spatial4j', version:'0.7' - compile group: "org.elasticsearch.plugin", name: 'parent-join-client', version: "${es_version}" - compile group: "org.elasticsearch.plugin", name: 'reindex-client', version: "${es_version}" - compile group: 'com.google.guava', name: 'guava', version:'15.0' - compile group: 'org.json', name: 'json', version:'20180813' - compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.9' - - // ANTLR gradle plugin and runtime dependency - antlr "org.antlr:antlr4:4.7.1" - compile "org.antlr:antlr4-runtime:4.7.1" - - //compileOnly group: 'org.locationtech.jts', name: 'jts-core', version:'1.15.0' - // compileOnly group: 'org.elasticsearch', name: 'elasticsearch', version:'6.5.3' - // compileOnly group: 'com.unboundid', name: 'unboundid-ldapsdk', version:'3.2.0' - // compileOnly group: 'org.bouncycastle', name: 'bcprov-jdk15on', version:'1.58' - // compileOnly group: 'log4j', name: 'log4j', version:'1.2.17' - // compileOnly group: 'org.apache.logging.log4j', name: 'log4j-api', version:'2.7' - // compileOnly group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.7' - compileOnly group: 'javax.servlet', name: 'servlet-api', version:'2.5' - - // testCompile group: 'org.hamcrest', name: 'hamcrest-all', version:'1.3' - // testCompile group: 'junit', name: 'junit', version:'4.11' - // testCompile group: 'com.alibaba', name: 'fastjson', version:'1.2.56' - // testCompile group: 'org.mockito', name: 'mockito-core', version:'2.23.4' - testCompile group: "org.elasticsearch.client", name: 'transport', version: "${es_version}" - - // JDBC drivers for comparison test. Somehow Apache Derby throws security permission exception. - testCompile group: 'com.amazon.opendistroforelasticsearch.client', name: 'opendistro-sql-jdbc', version: '1.3.0.0' - testCompile group: 'com.h2database', name: 'h2', version: '1.4.200' - testCompile group: 'org.xerial', name: 'sqlite-jdbc', version: '3.28.0' - //testCompile group: 'org.apache.derby', name: 'derby', version: '10.15.1.3' -} - -apply plugin: 'nebula.ospackage' - -// This is afterEvaluate because the bundlePlugin ZIP task is updated afterEvaluate and changes the ZIP name to match the plugin name -afterEvaluate { - ospackage { - packageName = "${name}" - release = isSnapshot ? "0.1" : '1' - version = "${project.version}" - "-SNAPSHOT" - - into '/usr/share/elasticsearch/plugins' - from(zipTree(bundlePlugin.archivePath)) { - into esplugin.name - } - - user 'root' - permissionGroup 'root' - fileMode 0644 - dirMode 0755 - - requires('elasticsearch-oss', versions.elasticsearch, EQUAL) - packager = 'Amazon' - vendor = 'Amazon' - os = 'LINUX' - prefix '/usr' - - license 'ASL-2.0' - maintainer 'OpenDistro for Elasticsearch Team ' - url 'https://opendistro.github.io/elasticsearch/downloads' - summary ''' - SQL plugin for OpenDistro for Elasticsearch. - Reference documentation can be found at https://opendistro.github.io/for-elasticsearch-docs/. - '''.stripIndent().replace('\n', ' ').trim() - } - - buildRpm { - arch = 'NOARCH' - archiveName "${packageName}-${version}.rpm" - dependsOn 'assemble' - } - - buildDeb { - arch = 'amd64' - archiveName "${packageName}-${version}.deb" - dependsOn 'assemble' - } - - task buildPackages(type: GradleBuild) { - tasks = ['build', 'buildRpm', 'buildDeb'] - } -} diff --git a/common/build.gradle b/common/build.gradle new file mode 100644 index 0000000000..0849fae080 --- /dev/null +++ b/common/build.gradle @@ -0,0 +1,15 @@ +plugins { + id 'java' + id "io.freefair.lombok" +} + +repositories { + mavenCentral() +} + +dependencies { + compile "org.antlr:antlr4-runtime:4.7.1" + compile group: 'com.google.guava', name: 'guava', version: '23.0' + + testCompile group: 'junit', name: 'junit', version: '4.12' +} diff --git a/common/lombok.config b/common/lombok.config new file mode 100644 index 0000000000..6aa51d71ec --- /dev/null +++ b/common/lombok.config @@ -0,0 +1,2 @@ +# This file is generated by the 'io.freefair.lombok' Gradle plugin +config.stopBubbling = true diff --git a/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/antlr/CaseInsensitiveCharStream.java b/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/antlr/CaseInsensitiveCharStream.java new file mode 100644 index 0000000000..f251076d6c --- /dev/null +++ b/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/antlr/CaseInsensitiveCharStream.java @@ -0,0 +1,83 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.common.antlr; + +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.misc.Interval; + +/** + * Custom stream to convert character to upper case for case insensitive grammar before sending to + * lexer. + */ +public class CaseInsensitiveCharStream implements CharStream { + + /** Character stream. */ + private final CharStream charStream; + + public CaseInsensitiveCharStream(String sql) { + this.charStream = CharStreams.fromString(sql); + } + + @Override + public String getText(Interval interval) { + return charStream.getText(interval); + } + + @Override + public void consume() { + charStream.consume(); + } + + @Override + public int LA(int i) { + int c = charStream.LA(i); + if (c <= 0) { + return c; + } + return Character.toUpperCase(c); + } + + @Override + public int mark() { + return charStream.mark(); + } + + @Override + public void release(int marker) { + charStream.release(marker); + } + + @Override + public int index() { + return charStream.index(); + } + + @Override + public void seek(int index) { + charStream.seek(index); + } + + @Override + public int size() { + return charStream.size(); + } + + @Override + public String getSourceName() { + return charStream.getSourceName(); + } +} diff --git a/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/antlr/SyntaxAnalysisErrorListener.java b/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/antlr/SyntaxAnalysisErrorListener.java new file mode 100644 index 0000000000..e5218ec66f --- /dev/null +++ b/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/antlr/SyntaxAnalysisErrorListener.java @@ -0,0 +1,77 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.common.antlr; + +import java.util.Locale; +import org.antlr.v4.runtime.BaseErrorListener; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Recognizer; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.misc.IntervalSet; + +/** + * Syntax analysis error listener that handles any syntax error by throwing exception with useful + * information. + */ +public class SyntaxAnalysisErrorListener extends BaseErrorListener { + + @Override + public void syntaxError( + Recognizer recognizer, + Object offendingSymbol, + int line, + int charPositionInLine, + String msg, + RecognitionException e) { + + CommonTokenStream tokens = (CommonTokenStream) recognizer.getInputStream(); + Token offendingToken = (Token) offendingSymbol; + String query = tokens.getText(); + + throw new SyntaxCheckException( + String.format( + Locale.ROOT, + "Failed to parse query due to offending symbol [%s] " + + "at: '%s' <--- HERE... More details: %s", + getOffendingText(offendingToken), + truncateQueryAtOffendingToken(query, offendingToken), + getDetails(recognizer, msg, e))); + } + + private String getOffendingText(Token offendingToken) { + return offendingToken.getText(); + } + + private String truncateQueryAtOffendingToken(String query, Token offendingToken) { + return query.substring(0, offendingToken.getStopIndex() + 1); + } + + /** + * As official JavaDoc says, e=null means parser was able to recover from the error. In other + * words, "msg" argument includes the information we want. + */ + private String getDetails(Recognizer recognizer, String msg, RecognitionException e) { + String details; + if (e == null) { + details = msg; + } else { + IntervalSet followSet = e.getExpectedTokens(); + details = "Expecting tokens in " + followSet.toString(recognizer.getVocabulary()); + } + return details; + } +} diff --git a/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/antlr/SyntaxCheckException.java b/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/antlr/SyntaxCheckException.java new file mode 100644 index 0000000000..6930c25bf5 --- /dev/null +++ b/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/antlr/SyntaxCheckException.java @@ -0,0 +1,22 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.common.antlr; + +public class SyntaxCheckException extends RuntimeException { + public SyntaxCheckException(String message) { + super(message); + } +} diff --git a/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/response/ResponseListener.java b/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/response/ResponseListener.java new file mode 100644 index 0000000000..e786de0bf4 --- /dev/null +++ b/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/response/ResponseListener.java @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.common.response; + +/** + * Response listener for response post-processing callback. This is necessary because execution + * engine may schedule and execute in different thread. + * + * @param response class + */ +public interface ResponseListener { + + /** + * Handle successful response. + * + * @param response successful response + */ + void onResponse(R response); + + /** + * Handle failed response. + * + * @param e exception captured + */ + void onFailure(Exception e); +} diff --git a/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/setting/Settings.java b/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/setting/Settings.java new file mode 100644 index 0000000000..c90b975a82 --- /dev/null +++ b/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/setting/Settings.java @@ -0,0 +1,39 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.common.setting; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +/** + * Setting. + */ +public abstract class Settings { + @RequiredArgsConstructor + public enum Key { + PPL_QUERY_MEMORY_LIMIT("opendistro.ppl.query.memory_limit"); + + @Getter + private final String keyValue; + } + + /** + * Get Setting Value. + */ + public abstract T getSettingValue(Key key); +} diff --git a/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/utils/StringUtils.java b/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/utils/StringUtils.java new file mode 100644 index 0000000000..1df4b26758 --- /dev/null +++ b/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/utils/StringUtils.java @@ -0,0 +1,52 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.common.utils; + +import com.google.common.base.Strings; + +public class StringUtils { + /** + * Unquote Identifier with mark. + * @param text string + * @param mark quotation mark + * @return An unquoted string whose outer pair of (single/double/back-tick) quotes have been + * removed + */ + public static String unquoteIdentifier(String text, String mark) { + if (isQuoted(text, mark)) { + return text.substring(mark.length(), text.length() - mark.length()); + } + return text; + } + + /** + * Unquote Identifier which has " or ' or ` as mark. + * @param text string + * @return An unquoted string whose outer pair of (single/double/back-tick) quotes have been + * removed + */ + public static String unquoteIdentifier(String text) { + if (isQuoted(text, "\"") || isQuoted(text, "'") || isQuoted(text, "`")) { + return text.substring(1, text.length() - 1); + } else { + return text; + } + } + + private static boolean isQuoted(String text, String mark) { + return !Strings.isNullOrEmpty(text) && text.startsWith(mark) && text.endsWith(mark); + } +} diff --git a/config/checkstyle/google_checks.xml b/config/checkstyle/google_checks.xml new file mode 100644 index 0000000000..4dfcaf39cb --- /dev/null +++ b/config/checkstyle/google_checks.xml @@ -0,0 +1,264 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/config/checkstyle/suppressions.xml b/config/checkstyle/suppressions.xml index ec4edb849d..bb76a38b97 100644 --- a/config/checkstyle/suppressions.xml +++ b/config/checkstyle/suppressions.xml @@ -5,6 +5,8 @@ - + + + \ No newline at end of file diff --git a/core/build.gradle b/core/build.gradle new file mode 100644 index 0000000000..acea197aec --- /dev/null +++ b/core/build.gradle @@ -0,0 +1,63 @@ +plugins { + id 'java' + id "io.freefair.lombok" + id 'jacoco' +} + +repositories { + mavenCentral() +} + +dependencies { + compile group: 'com.google.guava', name: 'guava', version: '23.0' + compile group: 'org.springframework', name: 'spring-context', version: '5.2.5.RELEASE' + compile group: 'org.springframework', name: 'spring-beans', version: '5.2.5.RELEASE' + compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.10' + compile project(':common') + + testImplementation('org.junit.jupiter:junit-jupiter:5.6.2') + testCompile group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' + testCompile group: 'org.springframework', name: 'spring-test', version: '5.2.5.RELEASE' + testCompile group: 'org.mockito', name: 'mockito-core', version: '3.3.3' + testCompile group: 'org.mockito', name: 'mockito-junit-jupiter', version: '3.3.3' +} + +test { + useJUnitPlatform() + testLogging { + events "passed", "skipped", "failed" + } +} + +jacoco { + toolVersion = "0.8.5" +} +jacocoTestReport { + reports { + html.enabled true + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, + exclude: ['**/ast/**']) + })) + } +} +test.finalizedBy(project.tasks.jacocoTestReport) +jacocoTestCoverageVerification { + violationRules { + rule { + limit { + minimum = 1.0 + } + + } + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, + exclude: ['**/ast/**']) + })) + } +} +check.dependsOn jacocoTestCoverageVerification \ No newline at end of file diff --git a/core/lombok.config b/core/lombok.config new file mode 100644 index 0000000000..189c0bef98 --- /dev/null +++ b/core/lombok.config @@ -0,0 +1,3 @@ +# This file is generated by the 'io.freefair.lombok' Gradle plugin +config.stopBubbling = true +lombok.addLombokGeneratedAnnotation = true diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalysisContext.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalysisContext.java new file mode 100644 index 0000000000..b04c88ef2d --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalysisContext.java @@ -0,0 +1,65 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.analysis; + +import java.util.Objects; + +/** + * The context used for Analyzer. + */ +public class AnalysisContext { + /** + * Environment stack for symbol scope management. + */ + private TypeEnvironment environment; + + public AnalysisContext() { + this.environment = new TypeEnvironment(null); + } + + public AnalysisContext(TypeEnvironment environment) { + this.environment = environment; + } + + /** + * Push a new environment. + */ + public void push() { + environment = new TypeEnvironment(environment); + } + + /** + * Return current environment. + * + * @return current environment + */ + public TypeEnvironment peek() { + return environment; + } + + /** + * Pop up current environment from environment chain. + * + * @return current environment (before pop) + */ + public TypeEnvironment pop() { + Objects.requireNonNull(environment, "Fail to pop context due to no environment present"); + + TypeEnvironment curEnv = environment; + environment = curEnv.getParent(); + return curEnv; + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/Analyzer.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/Analyzer.java new file mode 100644 index 0000000000..03202b34ff --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/Analyzer.java @@ -0,0 +1,251 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.analysis; + +import com.amazon.opendistroforelasticsearch.sql.analysis.symbol.Namespace; +import com.amazon.opendistroforelasticsearch.sql.analysis.symbol.Symbol; +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Argument; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Field; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Let; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Literal; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Map; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Aggregation; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Dedupe; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Eval; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Filter; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Project; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Relation; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Rename; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort.SortOption; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Values; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprMissingValue; +import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.LiteralExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalAggregation; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalDedupe; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalEval; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalFilter; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalProject; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRelation; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRemove; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRename; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalSort; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalValues; +import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; +import com.amazon.opendistroforelasticsearch.sql.storage.Table; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; + +/** + * Analyze the {@link UnresolvedPlan} in the {@link AnalysisContext} to construct the {@link + * LogicalPlan}. + */ +@RequiredArgsConstructor +public class Analyzer extends AbstractNodeVisitor { + private final ExpressionAnalyzer expressionAnalyzer; + private final StorageEngine storageEngine; + + public LogicalPlan analyze(UnresolvedPlan unresolved, AnalysisContext context) { + return unresolved.accept(this, context); + } + + @Override + public LogicalPlan visitRelation(Relation node, AnalysisContext context) { + context.push(); + TypeEnvironment curEnv = context.peek(); + Table table = storageEngine.getTable(node.getTableName()); + table.getFieldTypes().forEach((k, v) -> curEnv.define(new Symbol(Namespace.FIELD_NAME, k), v)); + return new LogicalRelation(node.getTableName()); + } + + @Override + public LogicalPlan visitFilter(Filter node, AnalysisContext context) { + LogicalPlan child = node.getChild().get(0).accept(this, context); + Expression condition = expressionAnalyzer.analyze(node.getCondition(), context); + return new LogicalFilter(child, condition); + } + + /** + * Build {@link LogicalRename}. + */ + @Override + public LogicalPlan visitRename(Rename node, AnalysisContext context) { + LogicalPlan child = node.getChild().get(0).accept(this, context); + ImmutableMap.Builder renameMapBuilder = + new ImmutableMap.Builder<>(); + for (Map renameMap : node.getRenameList()) { + Expression origin = expressionAnalyzer.analyze(renameMap.getOrigin(), context); + // We should define the new target field in the context instead of analyze it. + if (renameMap.getTarget() instanceof Field) { + ReferenceExpression target = + new ReferenceExpression(((Field) renameMap.getTarget()).getField().toString(), + origin.type()); + context.peek().define(target); + renameMapBuilder.put(DSL.ref(origin.toString(), origin.type()), target); + } else { + throw new SemanticCheckException( + String.format("the target expected to be field, but is %s", renameMap.getTarget())); + } + } + + return new LogicalRename(child, renameMapBuilder.build()); + } + + /** + * Build {@link LogicalAggregation}. + */ + @Override + public LogicalPlan visitAggregation(Aggregation node, AnalysisContext context) { + LogicalPlan child = node.getChild().get(0).accept(this, context); + ImmutableList.Builder aggregatorBuilder = new ImmutableList.Builder<>(); + for (UnresolvedExpression expr : node.getAggExprList()) { + aggregatorBuilder.add((Aggregator) expressionAnalyzer.analyze(expr, context)); + } + + ImmutableList.Builder groupbyBuilder = new ImmutableList.Builder<>(); + for (UnresolvedExpression expr : node.getGroupExprList()) { + groupbyBuilder.add(expressionAnalyzer.analyze(expr, context)); + } + return new LogicalAggregation(child, aggregatorBuilder.build(), groupbyBuilder.build()); + } + + /** + * Build {@link LogicalProject} or {@link LogicalRemove} from {@link Field}. + * + *

Todo, the include/exclude fields should change the env definition. The cons of current + * implementation is even the query contain the field reference which has been excluded from + * fields command. There is no {@link SemanticCheckException} will be thrown. Instead, the during + * runtime evaluation, the not exist field will be resolve to {@link ExprMissingValue} which will + * not impact the correctness. + * + *

Postpone the implementation when finding more use case. + */ + @Override + public LogicalPlan visitProject(Project node, AnalysisContext context) { + LogicalPlan child = node.getChild().get(0).accept(this, context); + + if (node.hasArgument()) { + Argument argument = node.getArgExprList().get(0); + Boolean exclude = (Boolean) argument.getValue().getValue(); + if (exclude) { + List referenceExpressions = + node.getProjectList().stream() + .map(expr -> (ReferenceExpression) expressionAnalyzer.analyze(expr, context)) + .collect(Collectors.toList()); + return new LogicalRemove(child, ImmutableSet.copyOf(referenceExpressions)); + } + } + + List expressions = node.getProjectList().stream() + .map(expr -> expressionAnalyzer.analyze(expr, context)) + .collect(Collectors.toList()); + return new LogicalProject(child, expressions); + } + + /** + * Build {@link LogicalEval}. + */ + @Override + public LogicalPlan visitEval(Eval node, AnalysisContext context) { + LogicalPlan child = node.getChild().get(0).accept(this, context); + ImmutableList.Builder> expressionsBuilder = + new Builder<>(); + for (Let let : node.getExpressionList()) { + Expression expression = expressionAnalyzer.analyze(let.getExpression(), context); + ReferenceExpression ref = DSL.ref(let.getVar().getField().toString(), expression.type()); + expressionsBuilder.add(ImmutablePair.of(ref, expression)); + TypeEnvironment typeEnvironment = context.peek(); + // define the new reference in type env. + typeEnvironment.define(ref); + } + return new LogicalEval(child, expressionsBuilder.build()); + } + + /** + * Build {@link LogicalSort}. + */ + @Override + public LogicalPlan visitSort(Sort node, AnalysisContext context) { + LogicalPlan child = node.getChild().get(0).accept(this, context); + // the first options is {"count": "integer"} + Integer count = (Integer) node.getOptions().get(0).getValue().getValue(); + List> sortList = + node.getSortList().stream() + .map( + sortField -> { + // the first options is {"asc": "true/false"} + Boolean asc = (Boolean) sortField.getFieldArgs().get(0).getValue().getValue(); + Expression expression = expressionAnalyzer.analyze(sortField, context); + return ImmutablePair.of( + asc ? SortOption.PPL_ASC : SortOption.PPL_DESC, expression); + }) + .collect(Collectors.toList()); + + return new LogicalSort(child, count, sortList); + } + + /** + * Build {@link LogicalDedupe}. + */ + @Override + public LogicalPlan visitDedupe(Dedupe node, AnalysisContext context) { + LogicalPlan child = node.getChild().get(0).accept(this, context); + List options = node.getOptions(); + // Todo, refactor the option. + Integer allowedDuplication = (Integer) options.get(0).getValue().getValue(); + Boolean keepEmpty = (Boolean) options.get(1).getValue().getValue(); + Boolean consecutive = (Boolean) options.get(2).getValue().getValue(); + + return new LogicalDedupe( + child, + node.getFields().stream() + .map(f -> expressionAnalyzer.analyze(f, context)) + .collect(Collectors.toList()), + allowedDuplication, + keepEmpty, + consecutive); + } + + @Override + public LogicalPlan visitValues(Values node, AnalysisContext context) { + List> values = node.getValues(); + List> valueExprs = new ArrayList<>(); + for (List value : values) { + valueExprs.add(value.stream() + .map(val -> (LiteralExpression) expressionAnalyzer.analyze(val, context)) + .collect(Collectors.toList())); + } + return new LogicalValues(valueExprs); + } + +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/ExpressionAnalyzer.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/ExpressionAnalyzer.java new file mode 100644 index 0000000000..dd1081bf52 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/ExpressionAnalyzer.java @@ -0,0 +1,157 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.analysis; + +import com.amazon.opendistroforelasticsearch.sql.analysis.symbol.Namespace; +import com.amazon.opendistroforelasticsearch.sql.analysis.symbol.Symbol; +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.AggregateFunction; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.And; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Compare; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.EqualTo; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Field; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Function; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Literal; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Not; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Or; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedAttribute; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Xor; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionName; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Analyze the {@link UnresolvedExpression} in the {@link AnalysisContext} to construct the {@link + * Expression}. + */ +public class ExpressionAnalyzer extends AbstractNodeVisitor { + private final BuiltinFunctionRepository repository; + private final DSL dsl; + + public ExpressionAnalyzer( + BuiltinFunctionRepository repository) { + this.repository = repository; + this.dsl = new DSL(repository); + } + + public Expression analyze(UnresolvedExpression unresolved, AnalysisContext context) { + return unresolved.accept(this, context); + } + + @Override + public Expression visitUnresolvedAttribute(UnresolvedAttribute node, AnalysisContext context) { + return visitIdentifier(node.getAttr(), context); + } + + @Override + public Expression visitEqualTo(EqualTo node, AnalysisContext context) { + Expression left = node.getLeft().accept(this, context); + Expression right = node.getRight().accept(this, context); + + return dsl.equal(left, right); + } + + @Override + public Expression visitLiteral(Literal node, AnalysisContext context) { + return DSL.literal(ExprValueUtils.fromObjectValue(node.getValue())); + } + + @Override + public Expression visitAnd(And node, AnalysisContext context) { + Expression left = node.getLeft().accept(this, context); + Expression right = node.getRight().accept(this, context); + + return dsl.and(left, right); + } + + @Override + public Expression visitOr(Or node, AnalysisContext context) { + Expression left = node.getLeft().accept(this, context); + Expression right = node.getRight().accept(this, context); + + return dsl.or(left, right); + } + + @Override + public Expression visitXor(Xor node, AnalysisContext context) { + Expression left = node.getLeft().accept(this, context); + Expression right = node.getRight().accept(this, context); + + return dsl.xor(left, right); + } + + @Override + public Expression visitNot(Not node, AnalysisContext context) { + return dsl.not(node.getExpression().accept(this, context)); + } + + @Override + public Expression visitAggregateFunction(AggregateFunction node, AnalysisContext context) { + Optional builtinFunctionName = BuiltinFunctionName.of(node.getFuncName()); + if (builtinFunctionName.isPresent()) { + Expression arg = node.getField().accept(this, context); + return (Aggregator) + repository.compile( + builtinFunctionName.get().getName(), Collections.singletonList(arg)); + } else { + throw new SemanticCheckException("Unsupported aggregation function " + node.getFuncName()); + } + } + + @Override + public Expression visitFunction(Function node, AnalysisContext context) { + FunctionName functionName = FunctionName.of(node.getFuncName()); + List arguments = + node.getFuncArgs().stream() + .map(unresolvedExpression -> analyze(unresolvedExpression, context)) + .collect(Collectors.toList()); + return (Expression) repository.compile(functionName, arguments); + } + + @Override + public Expression visitCompare(Compare node, AnalysisContext context) { + FunctionName functionName = FunctionName.of(node.getOperator()); + Expression left = analyze(node.getLeft(), context); + Expression right = analyze(node.getRight(), context); + return (Expression) + repository.compile(functionName, Arrays.asList(left, right)); + } + + @Override + public Expression visitField(Field node, AnalysisContext context) { + String attr = node.getField().toString(); + return visitIdentifier(attr, context); + } + + private Expression visitIdentifier(String ident, AnalysisContext context) { + TypeEnvironment typeEnv = context.peek(); + ReferenceExpression ref = DSL.ref(ident, + typeEnv.resolve(new Symbol(Namespace.FIELD_NAME, ident))); + return ref; + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/TypeEnvironment.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/TypeEnvironment.java new file mode 100644 index 0000000000..2262b05598 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/TypeEnvironment.java @@ -0,0 +1,84 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.analysis; + +import com.amazon.opendistroforelasticsearch.sql.analysis.symbol.Namespace; +import com.amazon.opendistroforelasticsearch.sql.analysis.symbol.Symbol; +import com.amazon.opendistroforelasticsearch.sql.analysis.symbol.SymbolTable; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment; +import java.util.Optional; +import lombok.Getter; + +/** + * The definition of Type Environment. + */ +public class TypeEnvironment implements Environment { + @Getter + private final TypeEnvironment parent; + private final SymbolTable symbolTable; + + public TypeEnvironment(TypeEnvironment parent) { + this.parent = parent; + this.symbolTable = new SymbolTable(); + } + + public TypeEnvironment(TypeEnvironment parent, SymbolTable symbolTable) { + this.parent = parent; + this.symbolTable = symbolTable; + } + + /** + * Resolve the {@link Expression} from environment. + * + * @param symbol Symbol + * @return resolved {@link ExprType} + */ + @Override + public ExprType resolve(Symbol symbol) { + for (TypeEnvironment cur = this; cur != null; cur = cur.parent) { + Optional typeOptional = cur.symbolTable.lookup(symbol); + if (typeOptional.isPresent()) { + return typeOptional.get(); + } + } + throw new SemanticCheckException( + String.format("can't resolve %s in type env", symbol)); + } + + /** + * Define symbol with the type. + * + * @param symbol symbol to define + * @param type type + */ + public void define(Symbol symbol, ExprType type) { + symbolTable.store(symbol, type); + } + + /** + * Define expression with the type. + * + * @param ref {@link ReferenceExpression} + */ + public void define(ReferenceExpression ref) { + define(new Symbol(Namespace.FIELD_NAME, ref.getAttr()), ref.type()); + } + +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/symbol/Namespace.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/symbol/Namespace.java new file mode 100644 index 0000000000..f7a17c5706 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/symbol/Namespace.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.analysis.symbol; + +/** + * Namespace of symbol to avoid naming conflict. + */ +public enum Namespace { + + FIELD_NAME("Field"), + FUNCTION_NAME("Function"); + + private final String name; + + Namespace(String name) { + this.name = name; + } + +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/symbol/Symbol.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/symbol/Symbol.java new file mode 100644 index 0000000000..b34ff0952a --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/symbol/Symbol.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.analysis.symbol; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Symbol in the scope. + */ +@ToString +@Getter +@RequiredArgsConstructor +public class Symbol { + private final Namespace namespace; + private final String name; +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/symbol/SymbolTable.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/symbol/SymbolTable.java new file mode 100644 index 0000000000..35acd06163 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/symbol/SymbolTable.java @@ -0,0 +1,90 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.analysis.symbol; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptyNavigableMap; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import java.util.EnumMap; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Optional; +import java.util.TreeMap; + +/** + * Symbol table for symbol definition and resolution. + */ +public class SymbolTable { + + /** + * Two-dimension hash table to manage symbols with type in different namespace. + */ + private Map> tableByNamespace = + new EnumMap<>(Namespace.class); + + /** + * Store symbol with the type. Create new map for namespace for the first time. + * + * @param symbol symbol to define + * @param type symbol type + */ + public void store(Symbol symbol, ExprType type) { + tableByNamespace.computeIfAbsent( + symbol.getNamespace(), + ns -> new TreeMap<>() + ).put(symbol.getName(), type); + } + + /** + * Look up symbol in the namespace map. + * + * @param symbol symbol to look up + * @return symbol type which is optional + */ + public Optional lookup(Symbol symbol) { + Map table = tableByNamespace.get(symbol.getNamespace()); + ExprType type = null; + if (table != null) { + type = table.get(symbol.getName()); + } + return Optional.ofNullable(type); + } + + /** + * Look up symbols by a prefix. + * + * @param prefix a symbol prefix + * @return symbols starting with the prefix + */ + public Map lookupByPrefix(Symbol prefix) { + NavigableMap table = tableByNamespace.get(prefix.getNamespace()); + if (table != null) { + return table.subMap(prefix.getName(), prefix.getName() + Character.MAX_VALUE); + } + return emptyMap(); + } + + /** + * Check if namespace map in empty (none definition). + * + * @param namespace a namespace + * @return true for empty + */ + public boolean isEmpty(Namespace namespace) { + return tableByNamespace.getOrDefault(namespace, emptyNavigableMap()).isEmpty(); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/AbstractNodeVisitor.java new file mode 100644 index 0000000000..cb962c241a --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/AbstractNodeVisitor.java @@ -0,0 +1,181 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast; + +import com.amazon.opendistroforelasticsearch.sql.ast.expression.AggregateFunction; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.And; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Argument; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.AttributeList; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Compare; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.EqualTo; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Field; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Function; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.In; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Let; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Literal; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Map; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Not; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Or; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.QualifiedName; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedAttribute; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Xor; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Aggregation; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Dedupe; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Eval; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Filter; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Project; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Relation; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Rename; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Values; + +/** + * AST nodes visitor Defines the traverse path. + */ +public abstract class AbstractNodeVisitor { + + public T visit(Node node, C context) { + return null; + } + + /** + * Visit child node. + * @param node {@link Node} + * @param context Context + * @return Return Type. + */ + public T visitChildren(Node node, C context) { + T result = defaultResult(); + + for (Node child : node.getChild()) { + T childResult = child.accept(this, context); + result = aggregateResult(result, childResult); + } + return result; + } + + private T defaultResult() { + return null; + } + + private T aggregateResult(T aggregate, T nextResult) { + return nextResult; + } + + public T visitRelation(Relation node, C context) { + return visitChildren(node, context); + } + + public T visitFilter(Filter node, C context) { + return visitChildren(node, context); + } + + public T visitProject(Project node, C context) { + return visitChildren(node, context); + } + + public T visitAggregation(Aggregation node, C context) { + return visitChildren(node, context); + } + + public T visitEqualTo(EqualTo node, C context) { + return visitChildren(node, context); + } + + public T visitLiteral(Literal node, C context) { + return visitChildren(node, context); + } + + public T visitUnresolvedAttribute(UnresolvedAttribute node, C context) { + return visitChildren(node, context); + } + + public T visitAttributeList(AttributeList node, C context) { + return visitChildren(node, context); + } + + public T visitMap(Map node, C context) { + return visitChildren(node, context); + } + + public T visitNot(Not node, C context) { + return visitChildren(node, context); + } + + public T visitOr(Or node, C context) { + return visitChildren(node, context); + } + + public T visitAnd(And node, C context) { + return visitChildren(node, context); + } + + public T visitXor(Xor node, C context) { + return visitChildren(node, context); + } + + public T visitAggregateFunction(AggregateFunction node, C context) { + return visitChildren(node, context); + } + + public T visitFunction(Function node, C context) { + return visitChildren(node, context); + } + + public T visitIn(In node, C context) { + return visitChildren(node, context); + } + + public T visitCompare(Compare node, C context) { + return visitChildren(node, context); + } + + public T visitArgument(Argument node, C context) { + return visitChildren(node, context); + } + + public T visitField(Field node, C context) { + return visitChildren(node, context); + } + + public T visitQualifiedName(QualifiedName node, C context) { + return visitChildren(node, context); + } + + public T visitRename(Rename node, C context) { + return visitChildren(node, context); + } + + public T visitEval(Eval node, C context) { + return visitChildren(node, context); + } + + public T visitLet(Let node, C context) { + return visitChildren(node, context); + } + + public T visitSort(Sort node, C context) { + return visitChildren(node, context); + } + + public T visitDedupe(Dedupe node, C context) { + return visitChildren(node, context); + } + + public T visitValues(Values node, C context) { + return visitChildren(node, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/Node.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/Node.java new file mode 100644 index 0000000000..e4701e09fa --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/Node.java @@ -0,0 +1,36 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast; + +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +/** + * AST node. + */ +@EqualsAndHashCode +@ToString +public abstract class Node { + + public R accept(AbstractNodeVisitor visitor, C context) { + return visitor.visitChildren(this, context); + } + + public List getChild() { + return null; + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java new file mode 100644 index 0000000000..b5096b4e60 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java @@ -0,0 +1,269 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.dsl; + +import com.amazon.opendistroforelasticsearch.sql.ast.expression.AggregateFunction; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.And; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Argument; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Compare; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.DataType; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.EqualTo; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Field; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Function; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.In; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Let; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Literal; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Map; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Not; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Or; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.QualifiedName; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedAttribute; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Xor; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Aggregation; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Dedupe; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Eval; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Filter; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Project; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Relation; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Rename; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Values; +import com.google.common.collect.ImmutableList; +import java.util.Arrays; +import java.util.List; +import lombok.experimental.UtilityClass; + +/** + * Class of static methods to create specific node instances. + */ +@UtilityClass +public class AstDSL { + + public static UnresolvedPlan filter(UnresolvedPlan input, UnresolvedExpression expression) { + return new Filter(expression).attach(input); + } + + public static UnresolvedPlan relation(String tableName) { + return new Relation(qualifiedName(tableName)); + } + + public static UnresolvedPlan project(UnresolvedPlan input, UnresolvedExpression... projectList) { + return new Project(Arrays.asList(projectList)).attach(input); + } + + public static Eval eval(UnresolvedPlan input, Let... projectList) { + return new Eval(Arrays.asList(projectList)).attach(input); + } + + public static UnresolvedPlan projectWithArg( + UnresolvedPlan input, List argList, UnresolvedExpression... projectList) { + return new Project(Arrays.asList(projectList), argList).attach(input); + } + + public static UnresolvedPlan agg( + UnresolvedPlan input, + List aggList, + List sortList, + List groupList, + List argList) { + return new Aggregation(aggList, sortList, groupList, argList).attach(input); + } + + public static UnresolvedPlan rename(UnresolvedPlan input, Map... maps) { + return new Rename(Arrays.asList(maps), input); + } + + /** + * Initialize Values node by rows of literals. + * @param values rows in which each row is a list of literal values + * @return Values node + */ + @SafeVarargs + public UnresolvedPlan values(List... values) { + return new Values(Arrays.asList(values)); + } + + public static UnresolvedExpression qualifiedName(String... parts) { + return new QualifiedName(Arrays.asList(parts)); + } + + public static UnresolvedExpression equalTo( + UnresolvedExpression left, UnresolvedExpression right) { + return new EqualTo(left, right); + } + + public static UnresolvedExpression unresolvedAttr(String attr) { + return new UnresolvedAttribute(attr); + } + + private static Literal literal(Object value, DataType type) { + return new Literal(value, type); + } + + public static Let let(Field var, UnresolvedExpression expression) { + return new Let(var, expression); + } + + public static Literal intLiteral(Integer value) { + return literal(value, DataType.INTEGER); + } + + public static Literal doubleLiteral(Double value) { + return literal(value, DataType.DOUBLE); + } + + public static Literal stringLiteral(String value) { + return literal(value, DataType.STRING); + } + + public static Literal booleanLiteral(Boolean value) { + return literal(value, DataType.BOOLEAN); + } + + public static Literal nullLiteral() { + return literal(null, DataType.NULL); + } + + public static Map map(String origin, String target) { + return new Map(new Field(origin), new Field(target)); + } + + public static Map map(UnresolvedExpression origin, UnresolvedExpression target) { + return new Map(origin, target); + } + + public static UnresolvedExpression aggregate(String func, UnresolvedExpression field) { + return new AggregateFunction(func, field); + } + + public static UnresolvedExpression aggregate( + String func, UnresolvedExpression field, UnresolvedExpression... args) { + return new AggregateFunction(func, field, Arrays.asList(args)); + } + + public static UnresolvedExpression function(String funcName, UnresolvedExpression... funcArgs) { + return new Function(funcName, Arrays.asList(funcArgs)); + } + + public static UnresolvedExpression not(UnresolvedExpression expression) { + return new Not(expression); + } + + public static UnresolvedExpression or(UnresolvedExpression left, UnresolvedExpression right) { + return new Or(left, right); + } + + public static UnresolvedExpression and(UnresolvedExpression left, UnresolvedExpression right) { + return new And(left, right); + } + + public static UnresolvedExpression xor(UnresolvedExpression left, UnresolvedExpression right) { + return new Xor(left, right); + } + + public static UnresolvedExpression in( + UnresolvedExpression field, UnresolvedExpression... valueList) { + return new In(field, Arrays.asList(valueList)); + } + + public static UnresolvedExpression compare( + String operator, UnresolvedExpression left, UnresolvedExpression right) { + return new Compare(operator, left, right); + } + + public static Argument argument(String argName, Literal argValue) { + return new Argument(argName, argValue); + } + + public static UnresolvedExpression field(UnresolvedExpression field) { + return new Field((QualifiedName) field); + } + + public static Field field(String field) { + return new Field(field); + } + + public static UnresolvedExpression field(UnresolvedExpression field, Argument... fieldArgs) { + return new Field((QualifiedName) field, Arrays.asList(fieldArgs)); + } + + public static Field field(String field, Argument... fieldArgs) { + return new Field(field, Arrays.asList(fieldArgs)); + } + + public static UnresolvedExpression field(UnresolvedExpression field, List fieldArgs) { + return new Field((QualifiedName) field, fieldArgs); + } + + public static Field field(String field, List fieldArgs) { + return new Field(field, fieldArgs); + } + + public static List exprList(UnresolvedExpression... exprList) { + return Arrays.asList(exprList); + } + + public static List exprList(Argument... exprList) { + return Arrays.asList(exprList); + } + + public static List defaultFieldsArgs() { + return exprList(argument("exclude", booleanLiteral(false))); + } + + /** + * Default Stats Command Args. + */ + public static List defaultStatsArgs() { + return exprList( + argument("partitions", intLiteral(1)), + argument("allnum", booleanLiteral(false)), + argument("delim", stringLiteral(" ")), + argument("dedupsplit", booleanLiteral(false))); + } + + /** + * Default Dedup Command Args. + */ + public static List defaultDedupArgs() { + return exprList( + argument("number", intLiteral(1)), + argument("keepempty", booleanLiteral(false)), + argument("consecutive", booleanLiteral(false))); + } + + public static List defaultSortOptions() { + return exprList(argument("count", intLiteral(1000)), argument("desc", booleanLiteral(false))); + } + + public static List sortOptions(int count) { + return exprList(argument("count", intLiteral(count)), argument("desc", booleanLiteral(false))); + } + + public static List defaultSortFieldArgs() { + return exprList(argument("asc", booleanLiteral(true)), argument("type", nullLiteral())); + } + + public static Sort sort(UnresolvedPlan input, List options, Field... sorts) { + return new Sort(input, options, Arrays.asList(sorts)); + } + + public static Dedupe dedupe(UnresolvedPlan input, List options, Field... fields) { + return new Dedupe(input, options, Arrays.asList(fields)); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/AggregateFunction.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/AggregateFunction.java new file mode 100644 index 0000000000..dac1445426 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/AggregateFunction.java @@ -0,0 +1,62 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.expression; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import java.util.Collections; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +/** + * Expression node of aggregate functions. + * Params include aggregate function name (AVG, SUM, MAX etc.) and the field to aggregate. + */ +@Getter +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class AggregateFunction extends UnresolvedExpression { + private final String funcName; + private final UnresolvedExpression field; + private final List argList; + + /** + * Constructor. + * @param funcName function name. + * @param field {@link UnresolvedExpression}. + */ + public AggregateFunction(String funcName, UnresolvedExpression field) { + this.funcName = funcName; + this.field = field; + this.argList = Collections.emptyList(); + } + + @Override + public List getChild() { + return Collections.singletonList(field); + } + + @Override + public R accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitAggregateFunction(this, context); + } + + @Override + public String toString() { + return String.format("%s(%s)", funcName, field); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/And.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/And.java new file mode 100644 index 0000000000..6bab1bd33e --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/And.java @@ -0,0 +1,46 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.expression; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import java.util.Arrays; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Expression node of logic AND. + */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class And extends UnresolvedExpression { + private final UnresolvedExpression left; + private final UnresolvedExpression right; + + @Override + public List getChild() { + return Arrays.asList(left, right); + } + + @Override + public R accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitAnd(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Argument.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Argument.java new file mode 100644 index 0000000000..7f16b16ece --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Argument.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.expression; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import java.util.Arrays; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Argument. + */ +@Getter +@ToString +@RequiredArgsConstructor +@EqualsAndHashCode(callSuper = false) +public class Argument extends UnresolvedExpression { + private final String argName; + private final Literal value; + + // private final DataType valueType; + @Override + public List getChild() { + return Arrays.asList(value); + } + + @Override + public R accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitArgument(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/AttributeList.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/AttributeList.java new file mode 100644 index 0000000000..b659365834 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/AttributeList.java @@ -0,0 +1,45 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.expression; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; + +/** + * Expression node that includes a list of Expression nodes. + */ +@ToString +@EqualsAndHashCode(callSuper = false) +@AllArgsConstructor +public class AttributeList extends UnresolvedExpression { + @Getter + private List attrList; + + @Override + public List getChild() { + return ImmutableList.copyOf(attrList); + } + + @Override + public R accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitAttributeList(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Compare.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Compare.java new file mode 100644 index 0000000000..5ba9383dc7 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Compare.java @@ -0,0 +1,44 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.expression; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import java.util.Arrays; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class Compare extends UnresolvedExpression { + private final String operator; + private final UnresolvedExpression left; + private final UnresolvedExpression right; + + @Override + public List getChild() { + return Arrays.asList(left, right); + } + + @Override + public R accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitCompare(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/DataType.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/DataType.java new file mode 100644 index 0000000000..51662e6c53 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/DataType.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.expression; + +public enum DataType { + TYPE_ERROR, + NULL, + + INTEGER, + DOUBLE, + STRING, + BOOLEAN +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/EqualTo.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/EqualTo.java new file mode 100644 index 0000000000..5d745314f0 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/EqualTo.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.expression; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import java.util.Arrays; +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; + +/** + * Expression node of binary operator or comparison relation EQUAL. + */ +@ToString +@EqualsAndHashCode(callSuper = false) +@AllArgsConstructor +public class EqualTo extends UnresolvedExpression { + @Getter + private UnresolvedExpression left; + @Getter + private UnresolvedExpression right; + + @Override + public List getChild() { + return Arrays.asList(left, right); + } + + @Override + public R accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitEqualTo(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Field.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Field.java new file mode 100644 index 0000000000..c8152f280d --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Field.java @@ -0,0 +1,61 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.expression; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.google.common.collect.ImmutableList; +import java.util.Collections; +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; + +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +@AllArgsConstructor +public class Field extends UnresolvedExpression { + private QualifiedName field; + private List fieldArgs = Collections.emptyList(); + + public Field(QualifiedName field) { + this.field = field; + } + + public Field(String field) { + this.field = new QualifiedName(field); + } + + public Field(String field, List fieldArgs) { + this.field = new QualifiedName(field); + this.fieldArgs = fieldArgs; + } + + public boolean hasArgument() { + return !fieldArgs.isEmpty(); + } + + @Override + public List getChild() { + return ImmutableList.of(this.field); + } + + @Override + public R accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitField(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Function.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Function.java new file mode 100644 index 0000000000..e4b12ea3a3 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Function.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.expression; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Expression node of scalar function. + * Params include function name (@funcName) and function arguments (@funcArgs) + */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class Function extends UnresolvedExpression { + private final String funcName; + private final List funcArgs; + + @Override + public List getChild() { + return ImmutableList.of(); + } + + @Override + public R accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitFunction(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/In.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/In.java new file mode 100644 index 0000000000..365787780b --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/In.java @@ -0,0 +1,49 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.expression; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import java.util.Arrays; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Expression node of one-to-many mapping relation IN. + * Params include the field expression and/or wildcard field expression, + * nested field expression (@field). + * And the values that the field is mapped to (@valueList). + */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class In extends UnresolvedExpression { + private final UnresolvedExpression field; + private final List valueList; + + @Override + public List getChild() { + return Arrays.asList(field); + } + + @Override + public R accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitIn(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Let.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Let.java new file mode 100644 index 0000000000..c4109e8b13 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Let.java @@ -0,0 +1,46 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.expression; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Represent the assign operation. e.g. velocity = distance/speed. + */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class Let extends UnresolvedExpression { + private final Field var; + private final UnresolvedExpression expression; + + @Override + public List getChild() { + return ImmutableList.of(); + } + + @Override + public R accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitLet(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Literal.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Literal.java new file mode 100644 index 0000000000..5a0f60175c --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Literal.java @@ -0,0 +1,49 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.expression; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Expression node of literal type + * Params include literal value (@value) and + * literal data type (@type) which can be selected from {@link DataType}. + */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class Literal extends UnresolvedExpression { + + private final Object value; + private final DataType type; + + @Override + public List getChild() { + return ImmutableList.of(); + } + + @Override + public R accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitLiteral(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Map.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Map.java new file mode 100644 index 0000000000..6fe1f307c5 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Map.java @@ -0,0 +1,46 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.expression; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import java.util.Arrays; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Expression node of one-to-one mapping relation. + */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class Map extends UnresolvedExpression { + private final UnresolvedExpression origin; + private final UnresolvedExpression target; + + @Override + public List getChild() { + return Arrays.asList(origin, target); + } + + @Override + public R accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitMap(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Not.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Not.java new file mode 100644 index 0000000000..1597b4fe55 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Not.java @@ -0,0 +1,45 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.expression; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import java.util.Arrays; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Expression node of the logic NOT. + */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class Not extends UnresolvedExpression { + private final UnresolvedExpression expression; + + @Override + public List getChild() { + return Arrays.asList(expression); + } + + @Override + public R accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitNot(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Or.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Or.java new file mode 100644 index 0000000000..8ed108f591 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Or.java @@ -0,0 +1,46 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.expression; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import java.util.Arrays; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Expression node of the logic OR. + */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class Or extends UnresolvedExpression { + private final UnresolvedExpression left; + private final UnresolvedExpression right; + + @Override + public List getChild() { + return Arrays.asList(left, right); + } + + @Override + public R accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitOr(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/QualifiedName.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/QualifiedName.java new file mode 100644 index 0000000000..3d1e35ff06 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/QualifiedName.java @@ -0,0 +1,94 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.expression; + +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.toList; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.google.common.collect.ImmutableList; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.StreamSupport; +import lombok.EqualsAndHashCode; +import lombok.Getter; + +@Getter +@EqualsAndHashCode(callSuper = false) +public class QualifiedName extends UnresolvedExpression { + private final List parts; + + public QualifiedName(String name) { + this.parts = Collections.singletonList(name); + } + + /** + * QualifiedName Constructor. + */ + public QualifiedName(Iterable parts) { + List partsList = StreamSupport.stream(parts.spliterator(), false).collect(toList()); + if (partsList.isEmpty()) { + throw new IllegalArgumentException("parts is empty"); + } + this.parts = partsList; + } + + /** + * Construct {@link QualifiedName} from list of string. + */ + public static QualifiedName of(String first, String... rest) { + requireNonNull(first); + ArrayList parts = new ArrayList<>(); + parts.add(first); + parts.addAll(Arrays.asList(rest)); + return new QualifiedName(parts); + } + + private static QualifiedName of(Iterable parts) { + return new QualifiedName(parts); + } + + /** + * Get Prefix of {@link QualifiedName}. + */ + public Optional getPrefix() { + if (parts.size() == 1) { + return Optional.empty(); + } + return Optional.of(QualifiedName.of(parts.subList(0, parts.size() - 1))); + } + + public String getSuffix() { + return parts.get(parts.size() - 1); + } + + public String toString() { + return String.join(".", this.parts); + } + + @Override + public List getChild() { + return ImmutableList.of(); + } + + @Override + public R accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitQualifiedName(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/UnresolvedAttribute.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/UnresolvedAttribute.java new file mode 100644 index 0000000000..a08e762e15 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/UnresolvedAttribute.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.expression; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Expression node, representing the syntax that is not resolved to + * any other expression nodes yet but non-negligible + * This expression is often created as the index name, field name etc. + */ +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +@Getter +public class UnresolvedAttribute extends UnresolvedExpression { + private final String attr; + + @Override + public List getChild() { + return ImmutableList.of(); + } + + @Override + public R accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitUnresolvedAttribute(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/UnresolvedExpression.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/UnresolvedExpression.java new file mode 100644 index 0000000000..e63aa167df --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/UnresolvedExpression.java @@ -0,0 +1,30 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.expression; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.amazon.opendistroforelasticsearch.sql.ast.Node; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +@EqualsAndHashCode(callSuper = false) +@ToString +public abstract class UnresolvedExpression extends Node { + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitChildren(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Xor.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Xor.java new file mode 100644 index 0000000000..8ce0b41e70 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Xor.java @@ -0,0 +1,46 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.expression; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import java.util.Arrays; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Expression node of the logic XOR. + */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class Xor extends UnresolvedExpression { + private final UnresolvedExpression left; + private final UnresolvedExpression right; + + @Override + public List getChild() { + return Arrays.asList(left, right); + } + + @Override + public R accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitXor(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Aggregation.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Aggregation.java new file mode 100644 index 0000000000..c996c205bb --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Aggregation.java @@ -0,0 +1,84 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.tree; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Argument; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; +import com.google.common.collect.ImmutableList; +import java.util.Collections; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +/** + * Logical plan node of Aggregation, the interface for building aggregation actions in queries. + */ +@Getter +@Setter +@ToString +@EqualsAndHashCode(callSuper = false) +public class Aggregation extends UnresolvedPlan { + private List aggExprList; + private List sortExprList; + private List groupExprList; + private List argExprList; + private UnresolvedPlan child; + + /** + * Aggregation Constructor without argument. + */ + public Aggregation(List aggExprList, + List sortExprList, + List groupExprList) { + this(aggExprList, sortExprList, groupExprList, Collections.emptyList()); + } + + /** + * Aggregation Constructor. + */ + public Aggregation(List aggExprList, + List sortExprList, + List groupExprList, + List argExprList) { + this.aggExprList = aggExprList; + this.sortExprList = sortExprList; + this.groupExprList = groupExprList; + this.argExprList = argExprList; + } + + public boolean hasArgument() { + return !aggExprList.isEmpty(); + } + + @Override + public Aggregation attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List getChild() { + return ImmutableList.of(this.child); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitAggregation(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Dedupe.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Dedupe.java new file mode 100644 index 0000000000..226c6eda7a --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Dedupe.java @@ -0,0 +1,59 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.tree; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Argument; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Field; +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +/** + * AST node represent Dedupe operation. + */ +@Getter +@Setter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +@AllArgsConstructor +public class Dedupe extends UnresolvedPlan { + private UnresolvedPlan child; + private final List options; + private final List fields; + + @Override + public Dedupe attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List getChild() { + return ImmutableList.of(this.child); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitDedupe(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Eval.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Eval.java new file mode 100644 index 0000000000..23ba92d815 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Eval.java @@ -0,0 +1,55 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.tree; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Let; +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +/** + * AST node represent Eval operation. + */ +@Getter +@Setter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class Eval extends UnresolvedPlan { + private final List expressionList; + private UnresolvedPlan child; + + @Override + public Eval attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List getChild() { + return ImmutableList.of(this.child); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitEval(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Filter.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Filter.java new file mode 100644 index 0000000000..7f8626b846 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Filter.java @@ -0,0 +1,55 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.tree; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; + +/** + * Logical plan node of Filter, the interface for building filters in queries. + */ +@ToString +@EqualsAndHashCode(callSuper = false) +@Getter +public class Filter extends UnresolvedPlan { + private UnresolvedExpression condition; + private UnresolvedPlan child; + + public Filter(UnresolvedExpression condition) { + this.condition = condition; + } + + @Override + public Filter attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List getChild() { + return ImmutableList.of(child); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitFilter(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Project.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Project.java new file mode 100644 index 0000000000..c11018192f --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Project.java @@ -0,0 +1,71 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.tree; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Argument; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; +import com.google.common.collect.ImmutableList; +import java.util.Collections; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +/** + * Logical plan node of Project, the interface for building the list of searching fields. + */ +@ToString +@Getter +@EqualsAndHashCode(callSuper = false) +public class Project extends UnresolvedPlan { + @Setter + private List projectList; + private List argExprList; + private UnresolvedPlan child; + + public Project(List projectList) { + this.projectList = projectList; + this.argExprList = Collections.emptyList(); + } + + public Project(List projectList, List argExprList) { + this.projectList = projectList; + this.argExprList = argExprList; + } + + public boolean hasArgument() { + return !argExprList.isEmpty(); + } + + @Override + public Project attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List getChild() { + return ImmutableList.of(this.child); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + + return nodeVisitor.visitProject(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Relation.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Relation.java new file mode 100644 index 0000000000..9fa002e177 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Relation.java @@ -0,0 +1,53 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.tree; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Logical plan node of Relation, the interface for building the searching sources. + */ +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class Relation extends UnresolvedPlan { + private final UnresolvedExpression tableName; + + public String getTableName() { + return tableName.toString(); + } + + @Override + public List getChild() { + return ImmutableList.of(); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitRelation(this, context); + } + + @Override + public UnresolvedPlan attach(UnresolvedPlan child) { + return this; + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Rename.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Rename.java new file mode 100644 index 0000000000..3576a93c36 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Rename.java @@ -0,0 +1,59 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.tree; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Map; +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +@ToString +@EqualsAndHashCode(callSuper = false) +@Getter +@RequiredArgsConstructor +public class Rename extends UnresolvedPlan { + private final List renameList; + private UnresolvedPlan child; + + public Rename(List renameList, UnresolvedPlan child) { + this.renameList = renameList; + this.child = child; + } + + @Override + public Rename attach(UnresolvedPlan child) { + if (null == this.child) { + this.child = child; + } else { + this.child.attach(child); + } + return this; + } + + @Override + public List getChild() { + return ImmutableList.of(child); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitRename(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Sort.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Sort.java new file mode 100644 index 0000000000..954a338e87 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Sort.java @@ -0,0 +1,92 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.tree; + +import static com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort.NullOrder.NULL_FIRST; +import static com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort.NullOrder.NULL_LAST; +import static com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort.SortOrder.ASC; +import static com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort.SortOrder.DESC; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Argument; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Field; +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * AST node for Sort {@link Sort#sortList} represent a list of sort expression and sort options. + */ +@ToString +@EqualsAndHashCode(callSuper = false) +@Getter +@RequiredArgsConstructor +@AllArgsConstructor +public class Sort extends UnresolvedPlan { + private UnresolvedPlan child; + private final List options; + private final List sortList; + + @Override + public Sort attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List getChild() { + return ImmutableList.of(child); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitSort(this, context); + } + + /** + * Sort Options. + */ + @Data + public static class SortOption { + + /** + * PPL ascending sort option, null first. + */ + public static SortOption PPL_ASC = new SortOption(ASC, NULL_FIRST); + /** + * PPL descending sort option, null last. + */ + public static SortOption PPL_DESC = new SortOption(DESC, NULL_LAST); + + private final SortOrder sortOrder; + private final NullOrder nullOrder; + } + + public enum SortOrder { + ASC, + DESC + } + + public enum NullOrder { + NULL_FIRST, + NULL_LAST + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/UnresolvedPlan.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/UnresolvedPlan.java new file mode 100644 index 0000000000..6e2e17c912 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/UnresolvedPlan.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.tree; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.amazon.opendistroforelasticsearch.sql.ast.Node; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +/** + * Abstract unresolved plan. + */ +@EqualsAndHashCode(callSuper = false) +@ToString +public abstract class UnresolvedPlan extends Node { + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitChildren(this, context); + } + + public abstract UnresolvedPlan attach(UnresolvedPlan child); +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Values.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Values.java new file mode 100644 index 0000000000..dc42644630 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Values.java @@ -0,0 +1,55 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.tree; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.amazon.opendistroforelasticsearch.sql.ast.Node; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Literal; +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * AST node class for a sequence of literal values. + */ +@ToString +@Getter +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class Values extends UnresolvedPlan { + + private final List> values; + + @Override + public UnresolvedPlan attach(UnresolvedPlan child) { + throw new UnsupportedOperationException("Values node is supposed to have no child node"); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitValues(this, context); + } + + @Override + public List getChild() { + return ImmutableList.of(); + } + +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprBooleanValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprBooleanValue.java new file mode 100644 index 0000000000..4d2abeb09a --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprBooleanValue.java @@ -0,0 +1,54 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import lombok.EqualsAndHashCode; + +@EqualsAndHashCode +public class ExprBooleanValue implements ExprValue { + private static final ExprValue TRUE = new ExprBooleanValue(true); + private static final ExprValue FALSE = new ExprBooleanValue(false); + + private final Boolean value; + + private ExprBooleanValue(Boolean value) { + this.value = value; + } + + public static ExprValue ofTrue() { + return TRUE; + } + + public static ExprValue ofFalse() { + return FALSE; + } + + @Override + public Object value() { + return value; + } + + @Override + public ExprCoreType type() { + return ExprCoreType.BOOLEAN; + } + + @Override + public String toString() { + return value.toString(); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprCollectionValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprCollectionValue.java new file mode 100644 index 0000000000..aa4053cef5 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprCollectionValue.java @@ -0,0 +1,45 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import java.util.List; +import java.util.stream.Collectors; +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; + +@EqualsAndHashCode +@RequiredArgsConstructor +public class ExprCollectionValue implements ExprValue { + private final List valueList; + + @Override + public Object value() { + return valueList; + } + + @Override + public ExprCoreType type() { + return ExprCoreType.ARRAY; + } + + @Override + public String toString() { + return valueList.stream() + .map(Object::toString) + .collect(Collectors.joining(",", "[", "]")); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDoubleValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDoubleValue.java new file mode 100644 index 0000000000..c464cbcfe3 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDoubleValue.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; + +@EqualsAndHashCode +@RequiredArgsConstructor +public class ExprDoubleValue implements ExprValue { + private final Double value; + + @Override + public Object value() { + return value; + } + + @Override + public ExprCoreType type() { + return ExprCoreType.DOUBLE; + } + + @Override + public String toString() { + return value.toString(); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprFloatValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprFloatValue.java new file mode 100644 index 0000000000..37c44058f0 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprFloatValue.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; + +@EqualsAndHashCode +@RequiredArgsConstructor +public class ExprFloatValue implements ExprValue { + private final Float value; + + @Override + public Object value() { + return value; + } + + @Override + public ExprCoreType type() { + return ExprCoreType.FLOAT; + } + + @Override + public String toString() { + return value.toString(); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprIntegerValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprIntegerValue.java new file mode 100644 index 0000000000..19d5462e78 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprIntegerValue.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; + +@EqualsAndHashCode +@RequiredArgsConstructor +public class ExprIntegerValue implements ExprValue { + private final Integer value; + + @Override + public Object value() { + return value; + } + + @Override + public ExprCoreType type() { + return ExprCoreType.INTEGER; + } + + @Override + public String toString() { + return value.toString(); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprLongValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprLongValue.java new file mode 100644 index 0000000000..730a7c406e --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprLongValue.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; + +@EqualsAndHashCode +@RequiredArgsConstructor +public class ExprLongValue implements ExprValue { + private final Long value; + + @Override + public Object value() { + return value; + } + + @Override + public ExprCoreType type() { + return ExprCoreType.LONG; + } + + @Override + public String toString() { + return value.toString(); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprMissingValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprMissingValue.java new file mode 100644 index 0000000000..3efe25f06c --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprMissingValue.java @@ -0,0 +1,50 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import lombok.EqualsAndHashCode; + +/** + * The definition of the expression missing value. + */ +@EqualsAndHashCode +public class ExprMissingValue implements ExprValue { + private static final ExprValue instance = new ExprMissingValue(); + + private ExprMissingValue() { + } + + public static ExprValue of() { + return instance; + } + + @Override + public Object value() { + throw new ExpressionEvaluationException("invalid to call value operation on missing value"); + } + + @Override + public ExprCoreType type() { + throw new ExpressionEvaluationException("invalid to call type operation on missing value"); + } + + @Override + public boolean isMissing() { + return true; + } +} \ No newline at end of file diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprNullValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprNullValue.java new file mode 100644 index 0000000000..b0b509c9e2 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprNullValue.java @@ -0,0 +1,50 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import lombok.EqualsAndHashCode; + +/** + * The definition of the expression null value. + */ +@EqualsAndHashCode +public class ExprNullValue implements ExprValue { + private static final ExprValue instance = new ExprNullValue(); + + private ExprNullValue() { + } + + public static ExprValue of() { + return instance; + } + + @Override + public Object value() { + return null; + } + + @Override + public ExprCoreType type() { + throw new ExpressionEvaluationException("invalid to call type operation on null value"); + } + + @Override + public boolean isNull() { + return true; + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprStringValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprStringValue.java new file mode 100644 index 0000000000..a06af303b5 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprStringValue.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; + +@EqualsAndHashCode +@RequiredArgsConstructor +public class ExprStringValue implements ExprValue { + private final String value; + + @Override + public Object value() { + return value; + } + + @Override + public ExprCoreType type() { + return ExprCoreType.STRING; + } + + @Override + public String toString() { + return String.format("\"%s\"", value); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTupleValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTupleValue.java new file mode 100644 index 0000000000..92df972f66 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTupleValue.java @@ -0,0 +1,84 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.BindingTuple; +import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.LazyBindingTuple; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class ExprTupleValue implements ExprValue { + + private final LinkedHashMap valueMap; + + public static ExprTupleValue fromExprValueMap(Map map) { + LinkedHashMap linkedHashMap = new LinkedHashMap<>(map); + return new ExprTupleValue(linkedHashMap); + } + + @Override + public Object value() { + return valueMap; + } + + @Override + public ExprCoreType type() { + return ExprCoreType.STRUCT; + } + + @Override + public String toString() { + return valueMap.entrySet() + .stream() + .map(entry -> String.format("%s:%s", entry.getKey(), entry.getValue())) + .collect(Collectors.joining(",", "{", "}")); + } + + @Override + public BindingTuple bindingTuples() { + return new LazyBindingTuple( + bindingName -> valueMap.getOrDefault(bindingName, ExprMissingValue.of())); + } + + /** + * Override the equals method. + * @return true for equal, otherwise false. + */ + public boolean equals(Object o) { + if (o == this) { + return true; + } else if (!(o instanceof ExprTupleValue)) { + return false; + } else { + ExprTupleValue other = (ExprTupleValue) o; + Iterator> thisIterator = this.valueMap.entrySet().iterator(); + Iterator> otherIterator = other.valueMap.entrySet().iterator(); + while (thisIterator.hasNext() && otherIterator.hasNext()) { + if (!thisIterator.next().equals(otherIterator.next())) { + return false; + } + } + return !(thisIterator.hasNext() || otherIterator.hasNext()); + } + } + +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValue.java new file mode 100644 index 0000000000..904b1aea09 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValue.java @@ -0,0 +1,59 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.BindingTuple; + +/** + * The definition of the Expression Value. + */ +public interface ExprValue { + /** + * Get the Object value of the Expression Value. + */ + Object value(); + + /** + * Get the {@link ExprCoreType} of the Expression Value. + */ + ExprCoreType type(); + + /** + * Is null value. + * + * @return true: is null value, otherwise false + */ + default boolean isNull() { + return false; + } + + /** + * Is missing value. + * + * @return true: is missing value, otherwise false + */ + default boolean isMissing() { + return false; + } + + /** + * Get the {@link BindingTuple}. + */ + default BindingTuple bindingTuples() { + return BindingTuple.EMPTY; + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java new file mode 100644 index 0000000000..1f601989fa --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java @@ -0,0 +1,181 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.ARRAY; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BOOLEAN; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRUCT; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import com.google.common.annotations.VisibleForTesting; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import lombok.experimental.UtilityClass; + +/** + * The definition of {@link ExprValue} factory. + */ +@UtilityClass +public class ExprValueUtils { + public static final ExprValue LITERAL_TRUE = ExprBooleanValue.ofTrue(); + public static final ExprValue LITERAL_FALSE = ExprBooleanValue.ofFalse(); + public static final ExprValue LITERAL_NULL = ExprNullValue.of(); + public static final ExprValue LITERAL_MISSING = ExprMissingValue.of(); + + public static ExprValue booleanValue(Boolean value) { + return value ? LITERAL_TRUE : LITERAL_FALSE; + } + + public static ExprValue integerValue(Integer value) { + return new ExprIntegerValue(value); + } + + public static ExprValue doubleValue(Double value) { + return new ExprDoubleValue(value); + } + + public static ExprValue floatValue(Float value) { + return new ExprFloatValue(value); + } + + public static ExprValue longValue(Long value) { + return new ExprLongValue(value); + } + + public static ExprValue stringValue(String value) { + return new ExprStringValue(value); + } + + /** + * {@link ExprTupleValue} constructor. + */ + public static ExprValue tupleValue(Map map) { + LinkedHashMap valueMap = new LinkedHashMap<>(); + map.forEach((k, v) -> valueMap.put(k, fromObjectValue(v))); + return new ExprTupleValue(valueMap); + } + + /** + * {@link ExprCollectionValue} constructor. + */ + public static ExprValue collectionValue(List list) { + List valueList = new ArrayList<>(); + list.forEach(o -> valueList.add(fromObjectValue(o))); + return new ExprCollectionValue(valueList); + } + + public static ExprValue missingValue() { + return ExprMissingValue.of(); + } + + public static ExprValue nullValue() { + return ExprNullValue.of(); + } + + /** + * Construct ExprValue from Object. + */ + public static ExprValue fromObjectValue(Object o) { + if (null == o) { + return LITERAL_NULL; + } + if (o instanceof Map) { + return tupleValue((Map) o); + } else if (o instanceof List) { + return collectionValue(((List) o)); + } else if (o instanceof Integer) { + return integerValue((Integer) o); + } else if (o instanceof Long) { + return longValue(((Long) o)); + } else if (o instanceof Boolean) { + return booleanValue((Boolean) o); + } else if (o instanceof Double) { + return doubleValue((Double) o); + } else if (o instanceof String) { + return stringValue((String) o); + } else if (o instanceof Float) { + return floatValue((Float) o); + } else { + throw new ExpressionEvaluationException("unsupported object " + o.getClass()); + } + } + + public static Integer getIntegerValue(ExprValue exprValue) { + return getNumberValue(exprValue).intValue(); + } + + public static Double getDoubleValue(ExprValue exprValue) { + return getNumberValue(exprValue).doubleValue(); + } + + public static Long getLongValue(ExprValue exprValue) { + return getNumberValue(exprValue).longValue(); + } + + public static Float getFloatValue(ExprValue exprValue) { + return getNumberValue(exprValue).floatValue(); + } + + public static String getStringValue(ExprValue exprValue) { + return convert(exprValue, STRING); + } + + public static List getCollectionValue(ExprValue exprValue) { + return convert(exprValue, ARRAY); + } + + public static Map getTupleValue(ExprValue exprValue) { + return convert(exprValue, STRUCT); + } + + public static Boolean getBooleanValue(ExprValue exprValue) { + return convert(exprValue, BOOLEAN); + } + + /** + * Get Number Value from {@link ExprValue}. + */ + @VisibleForTesting + public static Number getNumberValue(ExprValue exprValue) { + switch (exprValue.type()) { + case INTEGER: + case DOUBLE: + case LONG: + case FLOAT: + return (Number) exprValue.value(); + default: + break; + } + throw new ExpressionEvaluationException( + String + .format("invalid to getNumberValue with expression has type of %s", exprValue.type())); + } + + @SuppressWarnings("unchecked") + private static T convert(ExprValue exprValue, ExprCoreType toType) { + if (exprValue.type() == toType) { + return (T) exprValue.value(); + } else { + throw new ExpressionEvaluationException( + String.format("invalid to convert expression with type:%s to type:%s", exprValue.type(), + toType)); + } + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java new file mode 100644 index 0000000000..1e0d67149e --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java @@ -0,0 +1,90 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.data.type; + +import java.util.Arrays; +import java.util.List; + +/** + * Expression Type. + */ +public enum ExprCoreType implements ExprType { + /** + * UNKNOWN. + */ + UNKNOWN, + + /** + * Numbers. + */ + SHORT, + INTEGER(SHORT), + LONG(INTEGER), + FLOAT(LONG), + DOUBLE(FLOAT), + + /** + * Boolean. + */ + BOOLEAN, + + /** + * String. + */ + STRING, + + + /** + * Date. + * Todo. compatible relationship. + */ + TIMESTAMP, + DATE, + TIME, + + /** + * Struct. + */ + STRUCT, + + /** + * Array. + */ + ARRAY; + + /** + * Parent of current base type. + */ + private ExprCoreType parent; + + ExprCoreType(ExprCoreType... compatibleTypes) { + for (ExprCoreType subType : compatibleTypes) { + subType.parent = this; + } + } + + @Override + public List getParent() { + return Arrays.asList(parent == null ? UNKNOWN : parent); + } + + @Override + public String typeName() { + return this.name(); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprType.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprType.java new file mode 100644 index 0000000000..59aaae5040 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprType.java @@ -0,0 +1,61 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.data.type; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.UNKNOWN; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import java.util.Arrays; +import java.util.List; + +/** + * The Type of {@link Expression} and {@link ExprValue}. + */ +public interface ExprType { + /** + * Is compatible with other types. + */ + default boolean isCompatible(ExprType other) { + if (this.equals(other)) { + return true; + } else { + if (other.equals(UNKNOWN)) { + return false; + } + for (ExprType parentTypeOfOther : other.getParent()) { + if (isCompatible(parentTypeOfOther)) { + return true; + } + } + return false; + } + } + + /** + * Get the parent type. + */ + default List getParent() { + return Arrays.asList(UNKNOWN); + } + + /** + * Get the type name. + */ + String typeName(); +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/WideningTypeRule.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/WideningTypeRule.java new file mode 100644 index 0000000000..26d4950d1f --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/WideningTypeRule.java @@ -0,0 +1,87 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.data.type; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.UNKNOWN; + +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import lombok.experimental.UtilityClass; + +/** + * The definition of widening type rule for expression value. + * ExprType Widens to data types + * INTEGER LONG, FLOAT, DOUBLE + * LONG FLOAT, DOUBLE + * FLOAT DOUBLE + * DOUBLE DOUBLE + * STRING STRING + * BOOLEAN BOOLEAN + * ARRAY ARRAY + * STRUCT STRUCT + */ +@UtilityClass +public class WideningTypeRule { + public static final int IMPOSSIBLE_WIDENING = Integer.MAX_VALUE; + public static final int TYPE_EQUAL = 0; + + /** + * The widening distance is calculated from the leaf to root. + * e.g. distance(INTEGER, FLOAT) = 2, but distance(FLOAT, INTEGER) = IMPOSSIBLE_WIDENING + * + * @param type1 widen from type + * @param type2 widen to type + * @return The widening distance when widen one type to another type. + */ + public static int distance(ExprType type1, ExprType type2) { + return distance(type1, type2, TYPE_EQUAL); + } + + private static int distance(ExprType type1, ExprType type2, int distance) { + if (type1 == UNKNOWN) { + return IMPOSSIBLE_WIDENING; + } else if (type1 == type2) { + return distance; + } else { + return type1.getParent().stream() + .map(parentOfType1 -> distance(parentOfType1, type2, distance + 1)) + .reduce(Math::min).get(); + } + } + + /** + * The max type among two types. The max is defined as follow + * if type1 could widen to type2, then max is type2, vice versa + * if type1 could't widen to type2 and type2 could't widen to type1, + * then throw {@link ExpressionEvaluationException}. + * + * @param type1 type1 + * @param type2 type2 + * @return the max type among two types. + */ + public static ExprType max(ExprType type1, ExprType type2) { + int type1To2 = distance(type1, type2); + int type2To1 = distance(type2, type1); + + if (type1To2 == Integer.MAX_VALUE && type2To1 == Integer.MAX_VALUE) { + throw new ExpressionEvaluationException( + String.format("no max type of %s and %s ", type1, type2)); + } else { + return type1To2 == Integer.MAX_VALUE ? type1 : type2; + } + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprValueOrdering.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprValueOrdering.java new file mode 100644 index 0000000000..70cb751843 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprValueOrdering.java @@ -0,0 +1,50 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.utils; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.google.common.collect.Ordering; +import java.util.Comparator; +import lombok.RequiredArgsConstructor; + +/** + * Idea from guava {@link Ordering}. The only difference is the special logic to handle {@link + * com.amazon.opendistroforelasticsearch.sql.data.model.ExprNullValue} and {@link + * com.amazon.opendistroforelasticsearch.sql.data.model.ExprMissingValue} + */ +@RequiredArgsConstructor +public abstract class ExprValueOrdering implements Comparator { + + public static ExprValueOrdering natural() { + return NaturalExprValueOrdering.INSTANCE; + } + + public ExprValueOrdering reverse() { + return new ReverseExprValueOrdering(this); + } + + public ExprValueOrdering nullsFirst() { + return new NullsFirstExprValueOrdering(this); + } + + public ExprValueOrdering nullsLast() { + return new NullsLastExprValueOrdering(this); + } + + // Never make these public + static final int LEFT_IS_GREATER = 1; + static final int RIGHT_IS_GREATER = -1; +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/NaturalExprValueOrdering.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/NaturalExprValueOrdering.java new file mode 100644 index 0000000000..0aeb2562d2 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/NaturalExprValueOrdering.java @@ -0,0 +1,102 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.utils; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getBooleanValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getCollectionValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getDoubleValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getFloatValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getIntegerValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getLongValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getStringValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getTupleValue; +import static com.amazon.opendistroforelasticsearch.sql.expression.operator.OperatorUtils.COMPARE_WITH_NULL_OR_MISSING; +import static com.amazon.opendistroforelasticsearch.sql.expression.operator.OperatorUtils.LIST_COMPARATOR; +import static com.amazon.opendistroforelasticsearch.sql.expression.operator.OperatorUtils.MAP_COMPARATOR; +import static com.amazon.opendistroforelasticsearch.sql.expression.operator.OperatorUtils.STRING_COMPARATOR; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import com.google.common.collect.Ordering; + +/** + * Idea from guava {@link Ordering}. The only difference is the special logic to handle {@link + * com.amazon.opendistroforelasticsearch.sql.data.model.ExprNullValue} and {@link + * com.amazon.opendistroforelasticsearch.sql.data.model.ExprMissingValue} + */ +public class NaturalExprValueOrdering extends ExprValueOrdering { + static final ExprValueOrdering INSTANCE = new NaturalExprValueOrdering(); + + private transient ExprValueOrdering nullsFirst; + private transient ExprValueOrdering nullsLast; + + @Override + public int compare(ExprValue left, ExprValue right) { + if (COMPARE_WITH_NULL_OR_MISSING.test(left, right)) { + throw new ExpressionEvaluationException("compare with null or missing value is invalid"); + } + if (!left.type().equals(right.type())) { + throw new ExpressionEvaluationException( + String.format( + "compare expected value have same type, but with [%s, %s]", + left.type(), right.type())); + } + switch (left.type()) { + case DOUBLE: + return Double.compare(getDoubleValue(left), getDoubleValue(right)); + case FLOAT: + return Float.compare(getFloatValue(left), getFloatValue(right)); + case LONG: + return Long.compare(getLongValue(left), getLongValue(right)); + case INTEGER: + return Integer.compare(getIntegerValue(left), getIntegerValue(right)); + case BOOLEAN: + return Boolean.compare(getBooleanValue(left), getBooleanValue(right)); + case STRING: + return STRING_COMPARATOR.apply(getStringValue(left), getStringValue(right)); + case STRUCT: + return MAP_COMPARATOR.apply(getTupleValue(left), getTupleValue(right)); + case ARRAY: + return LIST_COMPARATOR.apply(getCollectionValue(left), getCollectionValue(right)); + default: + throw new ExpressionEvaluationException( + String.format("compare doesn't support type [%s]", left.type())); + } + } + + @Override + public ExprValueOrdering nullsFirst() { + ExprValueOrdering result = nullsFirst; + if (result == null) { + result = nullsFirst = super.nullsFirst(); + } + return result; + } + + @Override + public ExprValueOrdering nullsLast() { + ExprValueOrdering result = nullsLast; + if (result == null) { + result = nullsLast = super.nullsLast(); + } + return result; + } + + @Override + public ExprValueOrdering reverse() { + return super.reverse(); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/NullsFirstExprValueOrdering.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/NullsFirstExprValueOrdering.java new file mode 100644 index 0000000000..9cec956831 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/NullsFirstExprValueOrdering.java @@ -0,0 +1,59 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.utils; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.google.common.collect.Ordering; +import lombok.RequiredArgsConstructor; + +/** + * Idea from guava {@link Ordering}. The only difference is the special logic to handle {@link + * com.amazon.opendistroforelasticsearch.sql.data.model.ExprNullValue} and {@link + * com.amazon.opendistroforelasticsearch.sql.data.model.ExprMissingValue} + */ +@RequiredArgsConstructor +public class NullsFirstExprValueOrdering extends ExprValueOrdering { + private final ExprValueOrdering ordering; + + @Override + public int compare(ExprValue left, ExprValue right) { + if (left == right) { + return 0; + } + if (left.isNull() || left.isMissing()) { + return RIGHT_IS_GREATER; + } + if (right.isNull() || right.isMissing()) { + return LEFT_IS_GREATER; + } + return ordering.compare(left, right); + } + + @Override + public ExprValueOrdering reverse() { + return ordering.reverse().nullsLast(); + } + + @Override + public ExprValueOrdering nullsFirst() { + return this; + } + + @Override + public ExprValueOrdering nullsLast() { + return ordering.nullsLast(); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/NullsLastExprValueOrdering.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/NullsLastExprValueOrdering.java new file mode 100644 index 0000000000..e3091bde32 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/NullsLastExprValueOrdering.java @@ -0,0 +1,59 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.utils; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.google.common.collect.Ordering; +import lombok.RequiredArgsConstructor; + +/** + * Idea from guava {@link Ordering}. The only difference is the special logic to handle {@link + * com.amazon.opendistroforelasticsearch.sql.data.model.ExprNullValue} and {@link + * com.amazon.opendistroforelasticsearch.sql.data.model.ExprMissingValue} + */ +@RequiredArgsConstructor +public class NullsLastExprValueOrdering extends ExprValueOrdering { + private final ExprValueOrdering ordering; + + @Override + public int compare(ExprValue left, ExprValue right) { + if (left == right) { + return 0; + } + if (left.isNull() || left.isMissing()) { + return LEFT_IS_GREATER; + } + if (right.isNull() || right.isMissing()) { + return RIGHT_IS_GREATER; + } + return ordering.compare(left, right); + } + + @Override + public ExprValueOrdering reverse() { + return ordering.reverse().nullsFirst(); + } + + @Override + public ExprValueOrdering nullsFirst() { + return ordering.nullsFirst(); + } + + @Override + public ExprValueOrdering nullsLast() { + return this; + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ReverseExprValueOrdering.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ReverseExprValueOrdering.java new file mode 100644 index 0000000000..949113ccae --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ReverseExprValueOrdering.java @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.utils; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.google.common.collect.Ordering; +import lombok.RequiredArgsConstructor; + +/** + * Idea from guava {@link Ordering}. The only difference is the special logic to handle {@link + * com.amazon.opendistroforelasticsearch.sql.data.model.ExprNullValue} and {@link + * com.amazon.opendistroforelasticsearch.sql.data.model.ExprMissingValue} + */ +@RequiredArgsConstructor +public class ReverseExprValueOrdering extends ExprValueOrdering { + private final ExprValueOrdering forwardOrder; + + @Override + public int compare(ExprValue left, ExprValue right) { + return forwardOrder.compare(right, left); + } + + @Override + public ExprValueOrdering reverse() { + return forwardOrder; + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/exception/ExpressionEvaluationException.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/exception/ExpressionEvaluationException.java new file mode 100644 index 0000000000..040d0d522d --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/exception/ExpressionEvaluationException.java @@ -0,0 +1,25 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.exception; + +/** + * Exception for Expression Evaluation. + */ +public class ExpressionEvaluationException extends QueryEngineException { + public ExpressionEvaluationException(String message) { + super(message); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/exception/QueryEngineException.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/exception/QueryEngineException.java new file mode 100644 index 0000000000..2239d6e17c --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/exception/QueryEngineException.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.exception; + +/** + * Query analysis abstract exception. + */ +public class QueryEngineException extends RuntimeException { + + public QueryEngineException(String message) { + super(message); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/exception/SemanticCheckException.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/exception/SemanticCheckException.java new file mode 100644 index 0000000000..611e19d8c1 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/exception/SemanticCheckException.java @@ -0,0 +1,25 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.exception; + +/** + * Semantic Check Exception. + */ +public class SemanticCheckException extends QueryEngineException { + public SemanticCheckException(String message) { + super(message); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/ExecutionEngine.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/ExecutionEngine.java new file mode 100644 index 0000000000..ec6e5bfbf2 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/ExecutionEngine.java @@ -0,0 +1,46 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.executor; + +import com.amazon.opendistroforelasticsearch.sql.common.response.ResponseListener; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import java.util.List; +import lombok.Data; + +/** + * Execution engine that encapsulates execution details. + */ +public interface ExecutionEngine { + + /** + * Execute physical plan and call back response listener. + * + * @param plan executable physical plan + * @param listener response listener + */ + void execute(PhysicalPlan plan, ResponseListener listener); + + /** + * Data class that encapsulates ExprValue. + */ + @Data + class QueryResponse { + private final List results; + } + +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java new file mode 100644 index 0000000000..7af871dc53 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java @@ -0,0 +1,163 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; +import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository; +import java.util.Arrays; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class DSL { + private final BuiltinFunctionRepository repository; + + public static LiteralExpression literal(Integer value) { + return new LiteralExpression(ExprValueUtils.integerValue(value)); + } + + public static LiteralExpression literal(Long value) { + return new LiteralExpression(ExprValueUtils.longValue(value)); + } + + public static LiteralExpression literal(Float value) { + return new LiteralExpression(ExprValueUtils.floatValue(value)); + } + + public static LiteralExpression literal(Double value) { + return new LiteralExpression(ExprValueUtils.doubleValue(value)); + } + + public static LiteralExpression literal(String value) { + return new LiteralExpression(ExprValueUtils.stringValue(value)); + } + + public static LiteralExpression literal(Boolean value) { + return new LiteralExpression(ExprValueUtils.booleanValue(value)); + } + + public static LiteralExpression literal(ExprValue value) { + return new LiteralExpression(value); + } + + public static ReferenceExpression ref(String ref, ExprType type) { + return new ReferenceExpression(ref, type); + } + + public FunctionExpression abs(Expression... expressions) { + return (FunctionExpression) + repository.compile(BuiltinFunctionName.ABS.getName(), Arrays.asList(expressions)); + } + + public FunctionExpression add(Expression... expressions) { + return (FunctionExpression) + repository.compile(BuiltinFunctionName.ADD.getName(), Arrays.asList(expressions)); + } + + public FunctionExpression subtract(Expression... expressions) { + return (FunctionExpression) + repository.compile(BuiltinFunctionName.SUBTRACT.getName(), Arrays.asList(expressions)); + } + + public FunctionExpression multiply(Expression... expressions) { + return (FunctionExpression) + repository.compile(BuiltinFunctionName.MULTIPLY.getName(), Arrays.asList(expressions)); + } + + public FunctionExpression divide(Expression... expressions) { + return (FunctionExpression) + repository.compile(BuiltinFunctionName.DIVIDE.getName(), Arrays.asList(expressions)); + } + + public FunctionExpression module(Expression... expressions) { + return (FunctionExpression) + repository.compile(BuiltinFunctionName.MODULES.getName(), Arrays.asList(expressions)); + } + + public FunctionExpression and(Expression... expressions) { + return (FunctionExpression) + repository.compile(BuiltinFunctionName.AND.getName(), Arrays.asList(expressions)); + } + + public FunctionExpression or(Expression... expressions) { + return (FunctionExpression) + repository.compile(BuiltinFunctionName.OR.getName(), Arrays.asList(expressions)); + } + + public FunctionExpression xor(Expression... expressions) { + return (FunctionExpression) + repository.compile(BuiltinFunctionName.XOR.getName(), Arrays.asList(expressions)); + } + + public FunctionExpression not(Expression... expressions) { + return (FunctionExpression) + repository.compile(BuiltinFunctionName.NOT.getName(), Arrays.asList(expressions)); + } + + public FunctionExpression equal(Expression... expressions) { + return (FunctionExpression) + repository.compile(BuiltinFunctionName.EQUAL.getName(), Arrays.asList(expressions)); + } + + public FunctionExpression notequal(Expression... expressions) { + return (FunctionExpression) + repository.compile(BuiltinFunctionName.NOTEQUAL.getName(), Arrays.asList(expressions)); + } + + public FunctionExpression less(Expression... expressions) { + return (FunctionExpression) + repository.compile(BuiltinFunctionName.LESS.getName(), Arrays.asList(expressions)); + } + + public FunctionExpression lte(Expression... expressions) { + return (FunctionExpression) + repository.compile(BuiltinFunctionName.LTE.getName(), Arrays.asList(expressions)); + } + + public FunctionExpression greater(Expression... expressions) { + return (FunctionExpression) + repository.compile(BuiltinFunctionName.GREATER.getName(), Arrays.asList(expressions)); + } + + public FunctionExpression gte(Expression... expressions) { + return (FunctionExpression) + repository.compile(BuiltinFunctionName.GTE.getName(), Arrays.asList(expressions)); + } + + public FunctionExpression like(Expression... expressions) { + return (FunctionExpression) + repository.compile(BuiltinFunctionName.LIKE.getName(), Arrays.asList(expressions)); + } + + public Aggregator avg(Expression... expressions) { + return (Aggregator) + repository.compile(BuiltinFunctionName.AVG.getName(), Arrays.asList(expressions)); + } + + public Aggregator sum(Expression... expressions) { + return (Aggregator) + repository.compile(BuiltinFunctionName.SUM.getName(), Arrays.asList(expressions)); + } + + public Aggregator count(Expression... expressions) { + return (Aggregator) + repository.compile(BuiltinFunctionName.COUNT.getName(), Arrays.asList(expressions)); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/Expression.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/Expression.java new file mode 100644 index 0000000000..8f233cf224 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/Expression.java @@ -0,0 +1,36 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment; + +/** + * The definition of the resolved expression. + */ +public interface Expression { + + /** + * Evaluate the value of expression in the value environment. + */ + ExprValue valueOf(Environment valueEnv); + + /** + * The type of the expression. + */ + ExprType type(); +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/FunctionExpression.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/FunctionExpression.java new file mode 100644 index 0000000000..2969ee677f --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/FunctionExpression.java @@ -0,0 +1,38 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression; + +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionImplementation; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionName; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Function Expression. + */ +@EqualsAndHashCode +@RequiredArgsConstructor +@ToString +public abstract class FunctionExpression implements Expression, FunctionImplementation { + @Getter + private final FunctionName functionName; + + @Getter + private final List arguments; +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/LiteralExpression.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/LiteralExpression.java new file mode 100644 index 0000000000..685639e655 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/LiteralExpression.java @@ -0,0 +1,46 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment; +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; + +/** + * Literal Expression. + */ +@EqualsAndHashCode +@RequiredArgsConstructor +public class LiteralExpression implements Expression { + private final ExprValue exprValue; + + @Override + public ExprValue valueOf(Environment env) { + return exprValue; + } + + @Override + public ExprType type() { + return exprValue.type(); + } + + @Override + public String toString() { + return exprValue.toString(); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/ReferenceExpression.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/ReferenceExpression.java new file mode 100644 index 0000000000..6d5ba349e7 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/ReferenceExpression.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@EqualsAndHashCode +@RequiredArgsConstructor +public class ReferenceExpression implements Expression { + @Getter + private final String attr; + + private final ExprType type; + + @Override + public ExprValue valueOf(Environment env) { + return env.resolve(this); + } + + @Override + public ExprType type() { + return type; + } + + @Override + public String toString() { + return attr; + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/AggregationState.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/AggregationState.java new file mode 100644 index 0000000000..87f32d3f2c --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/AggregationState.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.aggregation; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.BindingTuple; + +/** + * Maintain the state when {@link Aggregator} iterate on the {@link BindingTuple}. + */ +public interface AggregationState { + /** + * Get {@link ExprValue} result. + */ + ExprValue result(); +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/Aggregator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/Aggregator.java new file mode 100644 index 0000000000..9d3653b4c7 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/Aggregator.java @@ -0,0 +1,73 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.aggregation; + +import com.amazon.opendistroforelasticsearch.sql.analysis.ExpressionAnalyzer; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionImplementation; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionName; +import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.BindingTuple; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +/** + * Aggregator which will iterate on the {@link BindingTuple}s to aggregate the result. + * The Aggregator is not well fit into Expression, because it has side effect. + * But we still want to make it implement {@link Expression} interface to make + * {@link ExpressionAnalyzer} easier. + */ +@EqualsAndHashCode +@RequiredArgsConstructor +public abstract class Aggregator + implements FunctionImplementation, Expression { + @Getter + private final FunctionName functionName; + @Getter + private final List arguments; + protected final ExprCoreType returnType; + + /** + * Create an {@link AggregationState} which will be used for aggregation. + */ + public abstract S create(); + + /** + * Iterate on the {@link BindingTuple}. + * + * @param tuple {@link BindingTuple} + * @param state {@link AggregationState} + * @return {@link AggregationState} + */ + public abstract S iterate(BindingTuple tuple, S state); + + @Override + public ExprValue valueOf(Environment valueEnv) { + throw new ExpressionEvaluationException( + String.format("can't evaluate on aggregator: %s", functionName)); + } + + @Override + public ExprType type() { + return returnType; + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/AggregatorFunction.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/AggregatorFunction.java new file mode 100644 index 0000000000..d75beae3f4 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/AggregatorFunction.java @@ -0,0 +1,109 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.aggregation; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.ARRAY; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BOOLEAN; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.FLOAT; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.LONG; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRUCT; + +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionBuilder; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionName; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionResolver; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionSignature; +import com.google.common.collect.ImmutableMap; +import java.util.Collections; +import lombok.experimental.UtilityClass; + +/** + * The definition of aggregator function + * avg, Accepts two numbers and produces a number. + * sum, Accepts two numbers and produces a number. + * max, Accepts two numbers and produces a number. + * min, Accepts two numbers and produces a number. + * count, Accepts two numbers and produces a number. + */ +@UtilityClass +public class AggregatorFunction { + /** + * Register Aggregation Function. + * @param repository {@link BuiltinFunctionRepository}. + */ + public static void register(BuiltinFunctionRepository repository) { + repository.register(avg()); + repository.register(sum()); + repository.register(count()); + } + + private static FunctionResolver avg() { + FunctionName functionName = BuiltinFunctionName.AVG.getName(); + return new FunctionResolver( + functionName, + new ImmutableMap.Builder() + .put(new FunctionSignature(functionName, Collections.singletonList(DOUBLE)), + arguments -> new AvgAggregator(arguments, DOUBLE)) + .build() + ); + } + + private static FunctionResolver count() { + FunctionName functionName = BuiltinFunctionName.COUNT.getName(); + return new FunctionResolver( + functionName, + new ImmutableMap.Builder() + .put(new FunctionSignature(functionName, Collections.singletonList(INTEGER)), + arguments -> new CountAggregator(arguments, INTEGER)) + .put(new FunctionSignature(functionName, Collections.singletonList(LONG)), + arguments -> new CountAggregator(arguments, INTEGER)) + .put(new FunctionSignature(functionName, Collections.singletonList(FLOAT)), + arguments -> new CountAggregator(arguments, INTEGER)) + .put(new FunctionSignature(functionName, Collections.singletonList(DOUBLE)), + arguments -> new CountAggregator(arguments, INTEGER)) + .put(new FunctionSignature(functionName, Collections.singletonList(STRING)), + arguments -> new CountAggregator(arguments, INTEGER)) + .put(new FunctionSignature(functionName, Collections.singletonList(STRUCT)), + arguments -> new CountAggregator(arguments, INTEGER)) + .put(new FunctionSignature(functionName, Collections.singletonList(ARRAY)), + arguments -> new CountAggregator(arguments, INTEGER)) + .put(new FunctionSignature(functionName, Collections.singletonList(BOOLEAN)), + arguments -> new CountAggregator(arguments, INTEGER)) + .build() + ); + } + + private static FunctionResolver sum() { + FunctionName functionName = BuiltinFunctionName.SUM.getName(); + return new FunctionResolver( + functionName, + new ImmutableMap.Builder() + .put(new FunctionSignature(functionName, Collections.singletonList(INTEGER)), + arguments -> new SumAggregator(arguments, INTEGER)) + .put(new FunctionSignature(functionName, Collections.singletonList(LONG)), + arguments -> new SumAggregator(arguments, LONG)) + .put(new FunctionSignature(functionName, Collections.singletonList(FLOAT)), + arguments -> new SumAggregator(arguments, FLOAT)) + .put(new FunctionSignature(functionName, Collections.singletonList(DOUBLE)), + arguments -> new SumAggregator(arguments, DOUBLE)) + .build() + ); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/AvgAggregator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/AvgAggregator.java new file mode 100644 index 0000000000..e8682c9db9 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/AvgAggregator.java @@ -0,0 +1,81 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.aggregation; + +import static com.amazon.opendistroforelasticsearch.sql.utils.ExpressionUtils.format; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprNullValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName; +import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.BindingTuple; +import java.util.List; +import java.util.Locale; + +/** + * The average aggregator aggregate the value evaluated by the expression. + * If the expression evaluated result is NULL or MISSING, then the result is NULL. + */ +public class AvgAggregator extends Aggregator { + + public AvgAggregator(List arguments, ExprCoreType returnType) { + super(BuiltinFunctionName.AVG.getName(), arguments, returnType); + } + + @Override + public AvgState create() { + return new AvgState(); + } + + @Override + public AvgState iterate(BindingTuple tuple, AvgState state) { + Expression expression = getArguments().get(0); + ExprValue value = expression.valueOf(tuple); + if (value.isNull() || value.isMissing()) { + state.isNullResult = true; + } else { + state.count++; + state.total += ExprValueUtils.getDoubleValue(value); + } + return state; + } + + @Override + public String toString() { + return String.format(Locale.ROOT, "avg(%s)", format(getArguments())); + } + + /** + * Average State. + */ + protected class AvgState implements AggregationState { + private int count; + private double total; + private boolean isNullResult = false; + + public AvgState() { + this.count = 0; + this.total = 0d; + } + + @Override + public ExprValue result() { + return isNullResult ? ExprNullValue.of() : ExprValueUtils.doubleValue(total / count); + } + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/CountAggregator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/CountAggregator.java new file mode 100644 index 0000000000..95c930fbae --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/CountAggregator.java @@ -0,0 +1,71 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.aggregation; + +import static com.amazon.opendistroforelasticsearch.sql.utils.ExpressionUtils.format; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.CountAggregator.CountState; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName; +import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.BindingTuple; +import java.util.List; +import java.util.Locale; + +public class CountAggregator extends Aggregator { + + public CountAggregator(List arguments, ExprCoreType returnType) { + super(BuiltinFunctionName.COUNT.getName(), arguments, returnType); + } + + @Override + public CountAggregator.CountState create() { + return new CountState(); + } + + @Override + public CountState iterate(BindingTuple tuple, CountState state) { + Expression expression = getArguments().get(0); + ExprValue value = expression.valueOf(tuple); + if (!(value.isNull() || value.isMissing())) { + state.count++; + } + return state; + } + + @Override + public String toString() { + return String.format(Locale.ROOT, "count(%s)", format(getArguments())); + } + + /** + * Count State. + */ + protected class CountState implements AggregationState { + private int count; + + public CountState() { + this.count = 0; + } + + @Override + public ExprValue result() { + return ExprValueUtils.integerValue(count); + } + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/SumAggregator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/SumAggregator.java new file mode 100644 index 0000000000..73a93d210d --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/SumAggregator.java @@ -0,0 +1,114 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.aggregation; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.doubleValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.floatValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getDoubleValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getFloatValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getIntegerValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getLongValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.integerValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.longValue; +import static com.amazon.opendistroforelasticsearch.sql.utils.ExpressionUtils.format; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprNullValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.SumAggregator.SumState; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName; +import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.BindingTuple; +import java.util.List; +import java.util.Locale; + +/** + * The sum aggregator aggregate the value evaluated by the expression. + * If the expression evaluated result is NULL or MISSING, then the result is NULL. + */ +public class SumAggregator extends Aggregator { + + public SumAggregator(List arguments, ExprCoreType returnType) { + super(BuiltinFunctionName.SUM.getName(), arguments, returnType); + } + + @Override + public SumState create() { + return new SumState(returnType); + } + + @Override + public SumState iterate(BindingTuple tuple, SumState state) { + Expression expression = getArguments().get(0); + ExprValue value = expression.valueOf(tuple); + if (value.isNull() || value.isMissing()) { + state.isNullResult = true; + } else { + state.add(value); + } + return state; + } + + @Override + public String toString() { + return String.format(Locale.ROOT, "sum(%s)", format(getArguments())); + } + + /** + * Sum State. + */ + protected class SumState implements AggregationState { + + private final ExprCoreType type; + private ExprValue sumResult; + private boolean isNullResult = false; + + public SumState(ExprCoreType type) { + this.type = type; + sumResult = ExprValueUtils.integerValue(0); + } + + /** + * Add value to current sumResult. + */ + public void add(ExprValue value) { + switch (type) { + case INTEGER: + sumResult = integerValue(getIntegerValue(sumResult) + getIntegerValue(value)); + break; + case LONG: + sumResult = longValue(getLongValue(sumResult) + getLongValue(value)); + break; + case FLOAT: + sumResult = floatValue(getFloatValue(sumResult) + getFloatValue(value)); + break; + case DOUBLE: + sumResult = doubleValue(getDoubleValue(sumResult) + getDoubleValue(value)); + break; + default: + throw new ExpressionEvaluationException( + String.format("unexpected type [%s] in sum aggregation", type)); + } + } + + @Override + public ExprValue result() { + return isNullResult ? ExprNullValue.of() : sumResult; + } + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/config/ExpressionConfig.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/config/ExpressionConfig.java new file mode 100644 index 0000000000..da9d9ea1d1 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/config/ExpressionConfig.java @@ -0,0 +1,53 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.config; + +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.AggregatorFunction; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository; +import com.amazon.opendistroforelasticsearch.sql.expression.operator.arthmetic.ArithmeticFunction; +import com.amazon.opendistroforelasticsearch.sql.expression.operator.arthmetic.UnaryFunction; +import com.amazon.opendistroforelasticsearch.sql.expression.operator.predicate.BinaryPredicateOperator; +import com.amazon.opendistroforelasticsearch.sql.expression.operator.predicate.UnaryPredicateOperator; +import java.util.HashMap; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Expression Config for Spring IoC. + */ +@Configuration +public class ExpressionConfig { + /** + * BuiltinFunctionRepository constructor. + */ + @Bean + public BuiltinFunctionRepository functionRepository() { + BuiltinFunctionRepository builtinFunctionRepository = + new BuiltinFunctionRepository(new HashMap<>()); + ArithmeticFunction.register(builtinFunctionRepository); + BinaryPredicateOperator.register(builtinFunctionRepository); + UnaryFunction.register(builtinFunctionRepository); + UnaryPredicateOperator.register(builtinFunctionRepository); + AggregatorFunction.register(builtinFunctionRepository); + return builtinFunctionRepository; + } + + @Bean + public DSL dsl(BuiltinFunctionRepository repository) { + return new DSL(repository); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/env/Environment.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/env/Environment.java new file mode 100644 index 0000000000..51ac2e0806 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/env/Environment.java @@ -0,0 +1,50 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.env; + +/** + * The definition of the environment. + * @param the type of expression + * @param the type of expression value + */ +public interface Environment { + + /** + * resolve the value of expression from the environment. + */ + V resolve(E var); + + /** + * Extend the environment. + * + * @param env environment + * @param expr expression. + * @param value expression value. + * @param the type of expression + * @param the type of expression value + * @return extended environment. + */ + static Environment extendEnv( + Environment env, E expr, V value) { + return var -> { + if (var.equals(expr)) { + return value; + } else { + return env.resolve(var); + } + }; + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionName.java new file mode 100644 index 0000000000..24bb3aafa5 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionName.java @@ -0,0 +1,64 @@ +package com.amazon.opendistroforelasticsearch.sql.expression.function; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import java.util.Optional; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +/** + * Builtin Function Name. + */ +@Getter +@RequiredArgsConstructor +public enum BuiltinFunctionName { + ABS(FunctionName.of("abs")), + TOSTRING(FunctionName.of("tostring")), + + /** + * Arithmetic Operators. + */ + ADD(FunctionName.of("+")), + SUBTRACT(FunctionName.of("-")), + MULTIPLY(FunctionName.of("*")), + DIVIDE(FunctionName.of("/")), + MODULES(FunctionName.of("%")), + + /** + * Boolean Operators. + */ + AND(FunctionName.of("and")), + OR(FunctionName.of("or")), + XOR(FunctionName.of("xor")), + NOT(FunctionName.of("not")), + EQUAL(FunctionName.of("=")), + NOTEQUAL(FunctionName.of("!=")), + LESS(FunctionName.of("<")), + LTE(FunctionName.of("<=")), + GREATER(FunctionName.of(">")), + GTE(FunctionName.of(">=")), + LIKE(FunctionName.of("like")), + + /** + * Aggregation Function. + */ + AVG(FunctionName.of("avg")), + SUM(FunctionName.of("sum")), + COUNT(FunctionName.of("count")); + + private final FunctionName name; + + private static final Map ALL_NATIVE_FUNCTIONS; + + static { + ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + for (BuiltinFunctionName func : BuiltinFunctionName.values()) { + builder.put(func.getName(), func); + } + ALL_NATIVE_FUNCTIONS = builder.build(); + } + + public static Optional of(String str) { + return Optional.ofNullable(ALL_NATIVE_FUNCTIONS.getOrDefault(FunctionName.of(str), null)); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionRepository.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionRepository.java new file mode 100644 index 0000000000..16a13b5267 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionRepository.java @@ -0,0 +1,50 @@ +package com.amazon.opendistroforelasticsearch.sql.expression.function; + +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; + +/** + * Builtin Function Repository. + */ +@RequiredArgsConstructor +public class BuiltinFunctionRepository { + private final Map functionResolverMap; + + /** + * Register {@link FunctionResolver} to the Builtin Function Repository. + * + * @param resolver {@link FunctionResolver} to be registered + */ + public void register(FunctionResolver resolver) { + functionResolverMap.put(resolver.getFunctionName(), resolver); + } + + /** + * Compile FunctionExpression. + */ + public FunctionImplementation compile(FunctionName functionName, List expressions) { + FunctionBuilder resolvedFunctionBuilder = resolve(new FunctionSignature(functionName, + expressions.stream().map(expression -> expression.type()).collect(Collectors.toList()))); + return resolvedFunctionBuilder.apply(expressions); + } + + /** + * Resolve the {@link FunctionBuilder} in Builtin Function Repository. + * + * @param functionSignature {@link FunctionSignature} + * @return {@link FunctionBuilder} + */ + public FunctionBuilder resolve(FunctionSignature functionSignature) { + FunctionName functionName = functionSignature.getFunctionName(); + if (functionResolverMap.containsKey(functionName)) { + return functionResolverMap.get(functionName).resolve(functionSignature); + } else { + throw new ExpressionEvaluationException( + String.format("unsupported function name: %s", functionName.getFunctionName())); + } + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionBuilder.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionBuilder.java new file mode 100644 index 0000000000..16353bb73d --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionBuilder.java @@ -0,0 +1,34 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.function; + +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import java.util.List; + +/** + * The definition of function which create {@link FunctionImplementation} + * from input {@link Expression} list. + */ +public interface FunctionBuilder { + + /** + * Create {@link FunctionImplementation} from input {@link Expression} list. + * + * @param arguments {@link Expression} list + * @return {@link FunctionImplementation} + */ + FunctionImplementation apply(List arguments); +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionImplementation.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionImplementation.java new file mode 100644 index 0000000000..9236e3dfae --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionImplementation.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.function; + +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import java.util.List; + +/** + * The definition of Function Implementation. + */ +public interface FunctionImplementation { + + /** + * Get Function Name. + */ + FunctionName getFunctionName(); + + /** + * Get Function Arguments. + */ + List getArguments(); +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionName.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionName.java new file mode 100644 index 0000000000..a3611a0506 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionName.java @@ -0,0 +1,39 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.function; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +/** + * The definition of Function Name. + */ +@EqualsAndHashCode +@RequiredArgsConstructor +public class FunctionName { + @Getter + private final String functionName; + + public static FunctionName of(String functionName) { + return new FunctionName(functionName); + } + + @Override + public String toString() { + return functionName; + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionResolver.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionResolver.java new file mode 100644 index 0000000000..7df7423c48 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionResolver.java @@ -0,0 +1,59 @@ +package com.amazon.opendistroforelasticsearch.sql.expression.function; + +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import java.util.AbstractMap; +import java.util.Map; +import java.util.PriorityQueue; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.Builder; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Singular; + +/** + * The Function Resolver hold the overload {@link FunctionBuilder} implementation. + * is composed by {@link FunctionName} which identified the function name + * and a map of {@link FunctionSignature} and {@link FunctionBuilder} + * to represent the overloaded implementation + */ +@Builder +@RequiredArgsConstructor +public class FunctionResolver { + @Getter + private final FunctionName functionName; + @Singular("functionBundle") + private final Map functionBundle; + + /** + * Resolve the {@link FunctionBuilder} by using input {@link FunctionSignature}. + * If the {@link FunctionBuilder} exactly match the input {@link FunctionSignature}, return it. + * If applying the widening rule, found the most match one, return it. + * If nothing found, throw {@link ExpressionEvaluationException} + */ + public FunctionBuilder resolve(FunctionSignature unresolvedSignature) { + PriorityQueue> functionMatchQueue = new PriorityQueue<>( + Map.Entry.comparingByKey()); + + for (FunctionSignature functionSignature : functionBundle.keySet()) { + functionMatchQueue.add( + new AbstractMap.SimpleEntry<>(unresolvedSignature.match(functionSignature), + functionSignature)); + } + Map.Entry bestMatchEntry = functionMatchQueue.peek(); + if (FunctionSignature.NOT_MATCH.equals(bestMatchEntry.getKey())) { + throw new ExpressionEvaluationException( + String.format("%s function expected %s, but get %s", functionName, + formatFunctions(functionBundle.keySet()), + unresolvedSignature.formatTypes() + )); + } else { + return functionBundle.get(bestMatchEntry.getValue()); + } + } + + private String formatFunctions(Set functionSignatures) { + return functionSignatures.stream().map(FunctionSignature::formatTypes) + .collect(Collectors.joining(",", "{", "}")); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionSignature.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionSignature.java new file mode 100644 index 0000000000..90bf7c8e4c --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionSignature.java @@ -0,0 +1,60 @@ +package com.amazon.opendistroforelasticsearch.sql.expression.function; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.data.type.WideningTypeRule; +import java.util.List; +import java.util.stream.Collectors; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +/** + * Function signature is composed by function name and arguments list. + */ +@Getter +@RequiredArgsConstructor +@EqualsAndHashCode +public class FunctionSignature { + public static final Integer NOT_MATCH = Integer.MAX_VALUE; + public static final Integer EXACTLY_MATCH = 0; + + private final FunctionName functionName; + private final List paramTypeList; + + /** + * calculate the function signature match degree. + * + * @return EXACTLY_MATCH: exactly match + * NOT_MATCH: not match + * By widening rule, the small number means better match + */ + public int match(FunctionSignature functionSignature) { + List functionTypeList = functionSignature.getParamTypeList(); + if (!functionName.equals(functionSignature.getFunctionName()) + || paramTypeList.size() != functionTypeList.size()) { + return NOT_MATCH; + } + + int matchDegree = EXACTLY_MATCH; + for (int i = 0; i < paramTypeList.size(); i++) { + ExprType paramType = paramTypeList.get(i); + ExprType funcType = functionTypeList.get(i); + int match = WideningTypeRule.distance(paramType, funcType); + if (match == WideningTypeRule.IMPOSSIBLE_WIDENING) { + return NOT_MATCH; + } else { + matchDegree += match; + } + } + return matchDegree; + } + + /** + * util function for formatted arguments list. + */ + public String formatTypes() { + return getParamTypeList().stream() + .map(ExprType::typeName) + .collect(Collectors.joining(",", "[", "]")); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/OperatorUtils.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/OperatorUtils.java new file mode 100644 index 0000000000..e8031c88ec --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/OperatorUtils.java @@ -0,0 +1,174 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.operator; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.FunctionExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionBuilder; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionName; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; +import java.util.function.BiPredicate; +import java.util.function.Function; +import java.util.stream.Collectors; +import lombok.experimental.UtilityClass; + +@UtilityClass +public class OperatorUtils { + /** + * Construct {@link FunctionBuilder} which call function with arguments produced by observer. + * + * @param functionName function name + * @param function {@link BiFunction} + * @param observer extract the value of type T from the first argument + * @param returnType return type + * @param the type of the first and second argument to the function + * @param the type of the result of the function + * @return {@link FunctionBuilder} + */ + public static FunctionBuilder binaryOperator( + FunctionName functionName, + BiFunction function, + Function observer, + ExprCoreType returnType) { + return binaryOperator(functionName, function, observer, observer, returnType); + } + + /** + * Construct {@link FunctionBuilder} which call function with arguments produced by observer1 and + * observer2 In general, if any operand evaluates to a MISSING value, the enclosing operator will + * return MISSING; if none of operands evaluates to a MISSING value but there is an operand + * evaluates to a NULL value, the enclosing operator will return NULL. + * + * @param functionName function name + * @param function {@link BiFunction} + * @param observer1 extract the value of type T from the first argument + * @param observer2 extract the value of type U from the second argument + * @param returnType return type + * @param the type of the first argument to the function + * @param the type of the second argument to the function + * @param the type of the result of the function + * @return {@link FunctionBuilder} + */ + public static FunctionBuilder binaryOperator( + FunctionName functionName, + BiFunction function, + Function observer1, + Function observer2, + ExprCoreType returnType) { + return arguments -> + new FunctionExpression(functionName, arguments) { + @Override + public ExprValue valueOf(Environment env) { + ExprValue arg1 = arguments.get(0).valueOf(env); + ExprValue arg2 = arguments.get(1).valueOf(env); + if (arg1.isMissing() || arg2.isMissing()) { + return ExprValueUtils.missingValue(); + } else if (arg1.isNull() || arg2.isNull()) { + return ExprValueUtils.nullValue(); + } else { + return ExprValueUtils.fromObjectValue( + function.apply(observer1.apply(arg1), observer2.apply(arg2))); + } + } + + @Override + public ExprType type() { + return returnType; + } + + @Override + public String toString() { + return String.format("%s %s %s", arguments.get(0).toString(), functionName, arguments + .get(1).toString()); + } + }; + } + + /** + * Construct {@link FunctionBuilder} which call function with arguments produced by observer In + * general, if any operand evaluates to a MISSING value, the enclosing operator will return + * MISSING; if none of operands evaluates to a MISSING value but there is an operand evaluates to + * a NULL value, the enclosing operator will return NULL. + * + * @param functionName function name + * @param function {@link Function} + * @param observer extract the value of type T from the first argument + * @param returnType return type + * @param the type of the first argument to the function + * @param the type of the result of the function + * @return {@link FunctionBuilder} + */ + public static FunctionBuilder unaryOperator( + FunctionName functionName, + Function function, + Function observer, + ExprCoreType returnType) { + return arguments -> + new FunctionExpression(functionName, arguments) { + @Override + public ExprValue valueOf(Environment env) { + ExprValue arg1 = arguments.get(0).valueOf(env); + if (arg1.isMissing()) { + return ExprValueUtils.missingValue(); + } else if (arg1.isNull()) { + return ExprValueUtils.nullValue(); + } else { + return ExprValueUtils.fromObjectValue(function.apply(observer.apply(arg1))); + } + } + + @Override + public ExprType type() { + return returnType; + } + + @Override + public String toString() { + return String.format("%s(%s)", functionName, + arguments.stream() + .map(Object::toString) + .collect(Collectors.joining(", "))); + } + }; + } + + /** + * String comparator. + */ + public static final BiFunction STRING_COMPARATOR = String::compareTo; + /** + * List comparator. + */ + public static final BiFunction LIST_COMPARATOR = + (left, right) -> Integer.compare(left.size(), right.size()); + /** + * Map comparator. + */ + public static final BiFunction MAP_COMPARATOR = + (left, right) -> Integer.compare(left.size(), right.size()); + /** + * Predicate NULL or MISSING. + */ + public static final BiPredicate COMPARE_WITH_NULL_OR_MISSING = + (left, right) -> left.isMissing() || right.isMissing() || left.isNull() || right.isNull(); +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/ArithmeticFunction.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/ArithmeticFunction.java new file mode 100644 index 0000000000..300b89065d --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/ArithmeticFunction.java @@ -0,0 +1,137 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.operator.arthmetic; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.FLOAT; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.LONG; +import static com.amazon.opendistroforelasticsearch.sql.expression.operator.OperatorUtils.binaryOperator; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionBuilder; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionName; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionResolver; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionSignature; +import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import java.util.Map; +import java.util.function.BiFunction; +import lombok.experimental.UtilityClass; + +/** + * The definition of arithmetic function + * add, Accepts two numbers and produces a number. + * subtract, Accepts two numbers and produces a number. + * multiply, Accepts two numbers and produces a number. + * divide, Accepts two numbers and produces a number. + * module, Accepts two numbers and produces a number. + */ +@UtilityClass +public class ArithmeticFunction { + /** + * Register Arithmetic Function. + * + * @param repository {@link BuiltinFunctionRepository}. + */ + public static void register(BuiltinFunctionRepository repository) { + repository.register(add()); + repository.register(subtract()); + repository.register(multiply()); + repository.register(divide()); + repository.register(modules()); + } + + private static FunctionResolver add() { + return new FunctionResolver( + BuiltinFunctionName.ADD.getName(), + scalarFunction(BuiltinFunctionName.ADD.getName(), + Math::addExact, + Math::addExact, + Float::sum, + Double::sum) + ); + } + + private static FunctionResolver subtract() { + return new FunctionResolver( + BuiltinFunctionName.SUBTRACT.getName(), + scalarFunction(BuiltinFunctionName.SUBTRACT.getName(), + Math::subtractExact, + Math::subtractExact, + (v1, v2) -> v1 - v2, + (v1, v2) -> v1 - v2) + ); + } + + private static FunctionResolver multiply() { + return new FunctionResolver( + BuiltinFunctionName.MULTIPLY.getName(), + scalarFunction(BuiltinFunctionName.MULTIPLY.getName(), + Math::multiplyExact, + Math::multiplyExact, + (v1, v2) -> v1 * v2, + (v1, v2) -> v1 * v2) + ); + } + + private static FunctionResolver divide() { + return new FunctionResolver( + BuiltinFunctionName.DIVIDE.getName(), + scalarFunction(BuiltinFunctionName.DIVIDE.getName(), + (v1, v2) -> v1 / v2, + (v1, v2) -> v1 / v2, + (v1, v2) -> v1 / v2, + (v1, v2) -> v1 / v2) + ); + } + + + private static FunctionResolver modules() { + return new FunctionResolver( + BuiltinFunctionName.MODULES.getName(), + scalarFunction(BuiltinFunctionName.MODULES.getName(), + (v1, v2) -> v1 % v2, + (v1, v2) -> v1 % v2, + (v1, v2) -> v1 % v2, + (v1, v2) -> v1 % v2) + ); + } + + private static Map scalarFunction( + FunctionName functionName, + BiFunction integerFunc, + BiFunction longFunc, + BiFunction floatFunc, + BiFunction doubleFunc) { + ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + builder + .put(new FunctionSignature(functionName, Arrays.asList(INTEGER, INTEGER)), + binaryOperator(functionName, integerFunc, ExprValueUtils::getIntegerValue, + INTEGER)); + builder.put(new FunctionSignature(functionName, Arrays.asList(LONG, LONG)), + binaryOperator(functionName, longFunc, ExprValueUtils::getLongValue, LONG)); + builder.put(new FunctionSignature(functionName, Arrays.asList(FLOAT, FLOAT)), + binaryOperator(functionName, floatFunc, ExprValueUtils::getFloatValue, FLOAT)); + builder + .put(new FunctionSignature(functionName, Arrays.asList(DOUBLE, DOUBLE)), + binaryOperator(functionName, doubleFunc, ExprValueUtils::getDoubleValue, + DOUBLE)); + return builder.build(); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/UnaryFunction.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/UnaryFunction.java new file mode 100644 index 0000000000..b6f0364570 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/UnaryFunction.java @@ -0,0 +1,81 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.operator.arthmetic; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.FLOAT; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.LONG; +import static com.amazon.opendistroforelasticsearch.sql.expression.operator.OperatorUtils.unaryOperator; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionBuilder; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionName; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionResolver; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionSignature; +import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import java.util.Map; +import java.util.function.Function; +import lombok.experimental.UtilityClass; + +@UtilityClass +public class UnaryFunction { + + public static void register(BuiltinFunctionRepository repository) { + repository.register(abs()); + } + + /** + * Definition of abs() function. + * The supported signature of abs() function are + * INT -> INT + * LONG -> LONG + * FLOAT -> FLOAT + * DOUBLE -> DOUBLE + */ + private static FunctionResolver abs() { + return new FunctionResolver( + BuiltinFunctionName.ABS.getName(), + unaryFunction( + BuiltinFunctionName.ABS.getName(), Math::abs, Math::abs, Math::abs, Math::abs)); + } + + private static Map unaryFunction( + FunctionName functionName, + Function integerFunc, + Function longFunc, + Function floatFunc, + Function doubleFunc) { + ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + builder.put( + new FunctionSignature(functionName, Arrays.asList(INTEGER)), + unaryOperator( + functionName, integerFunc, ExprValueUtils::getIntegerValue, INTEGER)); + builder.put( + new FunctionSignature(functionName, Arrays.asList(LONG)), + unaryOperator(functionName, longFunc, ExprValueUtils::getLongValue, LONG)); + builder.put( + new FunctionSignature(functionName, Arrays.asList(FLOAT)), + unaryOperator(functionName, floatFunc, ExprValueUtils::getFloatValue, FLOAT)); + builder.put( + new FunctionSignature(functionName, Arrays.asList(DOUBLE)), + unaryOperator(functionName, doubleFunc, ExprValueUtils::getDoubleValue, DOUBLE)); + return builder.build(); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java new file mode 100644 index 0000000000..d843fdc200 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java @@ -0,0 +1,503 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.operator.predicate; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.ARRAY; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BOOLEAN; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.FLOAT; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.LONG; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRUCT; +import static com.amazon.opendistroforelasticsearch.sql.expression.operator.OperatorUtils.binaryOperator; +import static com.amazon.opendistroforelasticsearch.sql.utils.OperatorUtils.matches; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.FunctionExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionBuilder; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionName; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionResolver; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionSignature; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableTable; +import com.google.common.collect.Table; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; +import java.util.function.Function; +import lombok.experimental.UtilityClass; + +/** + * The definition of binary predicate function + * and, Accepts two Boolean values and produces a Boolean. + * or, Accepts two Boolean values and produces a Boolean. + * xor, Accepts two Boolean values and produces a Boolean. + * equalTo, Compare the left expression and right expression and produces a Boolean. + */ +@UtilityClass +public class BinaryPredicateOperator { + /** + * Register Binary Predicate Function. + * + * @param repository {@link BuiltinFunctionRepository}. + */ + public static void register(BuiltinFunctionRepository repository) { + repository.register(and()); + repository.register(or()); + repository.register(xor()); + repository.register(equal()); + repository.register(notEqual()); + repository.register(less()); + repository.register(lte()); + repository.register(greater()); + repository.register(gte()); + repository.register(like()); + } + + /** + * The and logic. + * A B A AND B + * TRUE TRUE TRUE + * TRUE FALSE FALSE + * TRUE NULL NULL + * TRUE MISSING MISSING + * FALSE FALSE FALSE + * FALSE NULL FALSE + * FALSE MISSING FALSE + * NULL NULL NULL + * NULL MISSING MISSING + * MISSING MISSING MISSING + */ + private static Table andTable = + new ImmutableTable.Builder() + .put(LITERAL_TRUE, LITERAL_TRUE, LITERAL_TRUE) + .put(LITERAL_TRUE, LITERAL_FALSE, LITERAL_FALSE) + .put(LITERAL_TRUE, LITERAL_NULL, LITERAL_NULL) + .put(LITERAL_TRUE, LITERAL_MISSING, LITERAL_MISSING) + .put(LITERAL_FALSE, LITERAL_FALSE, LITERAL_FALSE) + .put(LITERAL_FALSE, LITERAL_NULL, LITERAL_FALSE) + .put(LITERAL_FALSE, LITERAL_MISSING, LITERAL_FALSE) + .put(LITERAL_NULL, LITERAL_NULL, LITERAL_NULL) + .put(LITERAL_NULL, LITERAL_MISSING, LITERAL_MISSING) + .put(LITERAL_MISSING, LITERAL_MISSING, LITERAL_MISSING) + .build(); + + /** + * The or logic. + * A B A AND B + * TRUE TRUE TRUE + * TRUE FALSE TRUE + * TRUE NULL TRUE + * TRUE MISSING TRUE + * FALSE FALSE FALSE + * FALSE NULL NULL + * FALSE MISSING MISSING + * NULL NULL NULL + * NULL MISSING NULL + * MISSING MISSING MISSING + */ + private static Table orTable = + new ImmutableTable.Builder() + .put(LITERAL_TRUE, LITERAL_TRUE, LITERAL_TRUE) + .put(LITERAL_TRUE, LITERAL_FALSE, LITERAL_TRUE) + .put(LITERAL_TRUE, LITERAL_NULL, LITERAL_TRUE) + .put(LITERAL_TRUE, LITERAL_MISSING, LITERAL_TRUE) + .put(LITERAL_FALSE, LITERAL_FALSE, LITERAL_FALSE) + .put(LITERAL_FALSE, LITERAL_NULL, LITERAL_NULL) + .put(LITERAL_FALSE, LITERAL_MISSING, LITERAL_MISSING) + .put(LITERAL_NULL, LITERAL_NULL, LITERAL_NULL) + .put(LITERAL_NULL, LITERAL_MISSING, LITERAL_NULL) + .put(LITERAL_MISSING, LITERAL_MISSING, LITERAL_MISSING) + .build(); + + /** + * The xor logic. + * A B A AND B + * TRUE TRUE FALSE + * TRUE FALSE TRUE + * TRUE NULL TRUE + * TRUE MISSING TRUE + * FALSE FALSE FALSE + * FALSE NULL NULL + * FALSE MISSING MISSING + * NULL NULL NULL + * NULL MISSING NULL + * MISSING MISSING MISSING + */ + private static Table xorTable = + new ImmutableTable.Builder() + .put(LITERAL_TRUE, LITERAL_TRUE, LITERAL_FALSE) + .put(LITERAL_TRUE, LITERAL_FALSE, LITERAL_TRUE) + .put(LITERAL_TRUE, LITERAL_NULL, LITERAL_TRUE) + .put(LITERAL_TRUE, LITERAL_MISSING, LITERAL_TRUE) + .put(LITERAL_FALSE, LITERAL_FALSE, LITERAL_FALSE) + .put(LITERAL_FALSE, LITERAL_NULL, LITERAL_NULL) + .put(LITERAL_FALSE, LITERAL_MISSING, LITERAL_MISSING) + .put(LITERAL_NULL, LITERAL_NULL, LITERAL_NULL) + .put(LITERAL_NULL, LITERAL_MISSING, LITERAL_NULL) + .put(LITERAL_MISSING, LITERAL_MISSING, LITERAL_MISSING) + .build(); + + /** + * The equalTo logic. + * A B A == B + * NULL NULL TRUE + * NULL MISSING FALSE + * MISSING NULL FALSE + * MISSING MISSING TRUE + */ + private static Table equalTable = + new ImmutableTable.Builder() + .put(LITERAL_NULL, LITERAL_NULL, LITERAL_TRUE) + .put(LITERAL_NULL, LITERAL_MISSING, LITERAL_FALSE) + .put(LITERAL_MISSING, LITERAL_NULL, LITERAL_FALSE) + .put(LITERAL_MISSING, LITERAL_MISSING, LITERAL_TRUE) + .build(); + + /** + * The notEqualTo logic. + * A B A != B + * NULL NULL FALSE + * NULL MISSING TRUE + * MISSING NULL TRUE + * MISSING MISSING FALSE + */ + private static Table notEqualTable = + new ImmutableTable.Builder() + .put(LITERAL_NULL, LITERAL_NULL, LITERAL_FALSE) + .put(LITERAL_NULL, LITERAL_MISSING, LITERAL_TRUE) + .put(LITERAL_MISSING, LITERAL_NULL, LITERAL_TRUE) + .put(LITERAL_MISSING, LITERAL_MISSING, LITERAL_FALSE) + .build(); + + private static FunctionResolver and() { + FunctionName functionName = BuiltinFunctionName.AND.getName(); + return FunctionResolver.builder() + .functionName(functionName) + .functionBundle(new FunctionSignature(functionName, + Arrays.asList(BOOLEAN, BOOLEAN)), binaryPredicate(functionName, + andTable, BOOLEAN)) + .build(); + } + + private static FunctionResolver or() { + FunctionName functionName = BuiltinFunctionName.OR.getName(); + return FunctionResolver.builder() + .functionName(functionName) + .functionBundle(new FunctionSignature(functionName, + Arrays.asList(BOOLEAN, BOOLEAN)), binaryPredicate(functionName, + orTable, BOOLEAN)) + .build(); + } + + private static FunctionResolver xor() { + FunctionName functionName = BuiltinFunctionName.XOR.getName(); + return FunctionResolver.builder() + .functionName(functionName) + .functionBundle(new FunctionSignature(functionName, + Arrays.asList(BOOLEAN, BOOLEAN)), binaryPredicate(functionName, + xorTable, BOOLEAN)) + .build(); + } + + private static FunctionResolver equal() { + return new FunctionResolver( + BuiltinFunctionName.EQUAL.getName(), + predicate( + BuiltinFunctionName.EQUAL.getName(), + equalTable, + LITERAL_FALSE, + Integer::equals, + Long::equals, + Float::equals, + Double::equals, + String::equals, + Boolean::equals, + List::equals, + Map::equals + ) + ); + } + + private static FunctionResolver notEqual() { + return new FunctionResolver( + BuiltinFunctionName.NOTEQUAL.getName(), + predicate( + BuiltinFunctionName.NOTEQUAL.getName(), + notEqualTable, + LITERAL_TRUE, + (v1, v2) -> ! v1.equals(v2), + (v1, v2) -> ! v1.equals(v2), + (v1, v2) -> ! v1.equals(v2), + (v1, v2) -> ! v1.equals(v2), + (v1, v2) -> ! v1.equals(v2), + (v1, v2) -> ! v1.equals(v2), + (v1, v2) -> ! v1.equals(v2), + (v1, v2) -> ! v1.equals(v2) + ) + ); + } + + private static FunctionResolver less() { + return new FunctionResolver( + BuiltinFunctionName.LESS.getName(), + predicate( + BuiltinFunctionName.LESS.getName(), + (v1, v2) -> v1 < v2, + (v1, v2) -> v1 < v2, + (v1, v2) -> v1 < v2, + (v1, v2) -> v1 < v2, + (v1, v2) -> v1.compareTo(v2) < 0 + ) + ); + } + + private static FunctionResolver lte() { + return new FunctionResolver( + BuiltinFunctionName.LTE.getName(), + predicate( + BuiltinFunctionName.LTE.getName(), + (v1, v2) -> v1 <= v2, + (v1, v2) -> v1 <= v2, + (v1, v2) -> v1 <= v2, + (v1, v2) -> v1 <= v2, + (v1, v2) -> v1.compareTo(v2) <= 0 + ) + ); + } + + private static FunctionResolver greater() { + return new FunctionResolver( + BuiltinFunctionName.GREATER.getName(), + predicate( + BuiltinFunctionName.GREATER.getName(), + (v1, v2) -> v1 > v2, + (v1, v2) -> v1 > v2, + (v1, v2) -> v1 > v2, + (v1, v2) -> v1 > v2, + (v1, v2) -> v1.compareTo(v2) > 0 + ) + ); + } + + private static FunctionResolver gte() { + return new FunctionResolver( + BuiltinFunctionName.GTE.getName(), + predicate( + BuiltinFunctionName.GTE.getName(), + (v1, v2) -> v1 >= v2, + (v1, v2) -> v1 >= v2, + (v1, v2) -> v1 >= v2, + (v1, v2) -> v1 >= v2, + (v1, v2) -> v1.compareTo(v2) >= 0 + ) + ); + } + + private static FunctionResolver like() { + return new FunctionResolver( + BuiltinFunctionName.LIKE.getName(), + predicate( + BuiltinFunctionName.LIKE.getName(), + (v1, v2) -> matches(v2, v1) + ) + ); + } + + /** + * Util method to generate EQUAL/NOT EQUAL operation bundles. + * Applicable for integer, long, float, double, string types of operands + * {@param defaultValue} Default value for one missing/null operand + */ + private static Map predicate( + FunctionName functionName, + Table table, + ExprValue defaultValue, + BiFunction integerFunc, + BiFunction longFunc, + BiFunction floatFunc, + BiFunction doubleFunc, + BiFunction stringFunc, + BiFunction booleanFunc, + BiFunction listFunc, + BiFunction mapFunc) { + ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + return builder + .put(new FunctionSignature(functionName, Arrays.asList(INTEGER, INTEGER)), + equalPredicate(functionName, table, integerFunc, ExprValueUtils::getIntegerValue, + defaultValue, BOOLEAN)) + .put(new FunctionSignature(functionName, Arrays.asList(LONG, LONG)), + equalPredicate(functionName, table, longFunc, ExprValueUtils::getLongValue, + defaultValue, BOOLEAN)) + .put(new FunctionSignature(functionName, Arrays.asList(FLOAT, FLOAT)), + equalPredicate(functionName, table, floatFunc, ExprValueUtils::getFloatValue, + defaultValue, BOOLEAN)) + .put(new FunctionSignature(functionName, Arrays.asList(DOUBLE, DOUBLE)), + equalPredicate(functionName, table, doubleFunc, ExprValueUtils::getDoubleValue, + defaultValue, BOOLEAN)) + .put(new FunctionSignature(functionName, Arrays.asList(STRING, STRING)), + equalPredicate(functionName, table, stringFunc, ExprValueUtils::getStringValue, + defaultValue, BOOLEAN)) + .put(new FunctionSignature(functionName, Arrays.asList(BOOLEAN, BOOLEAN)), + equalPredicate(functionName, table, booleanFunc, ExprValueUtils::getBooleanValue, + defaultValue, BOOLEAN)) + .put(new FunctionSignature(functionName, Arrays.asList(ARRAY, ARRAY)), + equalPredicate(functionName, table, listFunc, ExprValueUtils::getCollectionValue, + defaultValue, BOOLEAN)) + .put(new FunctionSignature(functionName, Arrays.asList(STRUCT, STRUCT)), + equalPredicate(functionName, table, mapFunc, ExprValueUtils::getTupleValue, + defaultValue, BOOLEAN)) + .build(); + } + + /** + * Util method to generate binary predicate bundles. + * Applicable for integer, long, float, double, string types of operands + * Missing/Null value operands follow as {@param table} lists + */ + private static Map predicate( + FunctionName functionName, + BiFunction integerFunc, + BiFunction longFunc, + BiFunction floatFunc, + BiFunction doubleFunc, + BiFunction stringFunc) { + ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + return builder + .put( + new FunctionSignature(functionName, Arrays.asList(INTEGER, INTEGER)), + binaryOperator( + functionName, integerFunc, ExprValueUtils::getIntegerValue, BOOLEAN)) + .put( + new FunctionSignature(functionName, Arrays.asList(LONG, LONG)), + binaryOperator( + functionName, longFunc, ExprValueUtils::getLongValue, BOOLEAN)) + .put( + new FunctionSignature(functionName, Arrays.asList(FLOAT, FLOAT)), + binaryOperator( + functionName, floatFunc, ExprValueUtils::getFloatValue, BOOLEAN)) + .put( + new FunctionSignature(functionName, Arrays.asList(DOUBLE, DOUBLE)), + binaryOperator( + functionName, doubleFunc, ExprValueUtils::getDoubleValue, BOOLEAN)) + .put( + new FunctionSignature(functionName, Arrays.asList(STRING, STRING)), + binaryOperator( + functionName, stringFunc, ExprValueUtils::getStringValue, BOOLEAN)) + .build(); + } + + /** + * Util method to generate LIKE predicate bundles. + * Applicable for string operands. + */ + private static Map predicate( + FunctionName functionName, + BiFunction stringFunc) { + ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + return builder + .put(new FunctionSignature(functionName, Arrays.asList(STRING, STRING)), + binaryOperator(functionName, stringFunc, ExprValueUtils::getStringValue, + BOOLEAN)) + .build(); + } + + + /** + * Building method to construct binary logical predicates AND OR XOR + * Where operands order does not matter. + * Special cases for missing/null operands refer to {@param table}. + */ + private static FunctionBuilder binaryPredicate(FunctionName functionName, + Table table, + ExprCoreType returnType) { + return arguments -> new FunctionExpression(functionName, arguments) { + @Override + public ExprValue valueOf(Environment env) { + ExprValue arg1 = arguments.get(0).valueOf(env); + ExprValue arg2 = arguments.get(1).valueOf(env); + if (table.contains(arg1, arg2)) { + return table.get(arg1, arg2); + } else { + return table.get(arg2, arg1); + } + } + + @Override + public ExprType type() { + return returnType; + } + + @Override + public String toString() { + return String.format("%s %s %s", arguments.get(0).toString(), functionName, arguments + .get(1).toString()); + } + }; + } + + /** + * Building method for equalTo and notEqualTo operators. + * + * @param defaultValue the return value when expr value is missing/null + */ + private static FunctionBuilder equalPredicate(FunctionName functionName, + Table table, + BiFunction function, + Function observer, + ExprValue defaultValue, + ExprType returnType) { + return arguments -> new FunctionExpression(functionName, arguments) { + @Override + public ExprValue valueOf(Environment env) { + ExprValue arg1 = arguments.get(0).valueOf(env); + ExprValue arg2 = arguments.get(1).valueOf(env); + if (table.contains(arg1, arg2)) { + return table.get(arg1, arg2); + } else if (arg1.isMissing() || arg1.isNull() || arg2.isMissing() || arg2.isNull()) { + return defaultValue; + } else { + return ExprValueUtils.fromObjectValue( + function.apply(observer.apply(arg1), observer.apply(arg2))); + } + } + + @Override + public ExprType type() { + return returnType; + } + + @Override + public String toString() { + return String.format("%s %s %s", arguments.get(0).toString(), functionName, arguments + .get(1).toString()); + } + }; + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/UnaryPredicateOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/UnaryPredicateOperator.java new file mode 100644 index 0000000000..f849a72bff --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/UnaryPredicateOperator.java @@ -0,0 +1,95 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.operator.predicate; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BOOLEAN; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.FunctionExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionBuilder; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionName; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionResolver; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionSignature; +import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import java.util.Map; +import lombok.experimental.UtilityClass; + +/** + * The definition of unary predicate function + * not, Accepts one Boolean value and produces a Boolean. + */ +@UtilityClass +public class UnaryPredicateOperator { + public static void register(BuiltinFunctionRepository repository) { + repository.register(not()); + } + + /** + * The not logic. + * A NOT A + * TRUE FALSE + * FALSE TRUE + * NULL NULL + * MISSING MISSING + */ + private static Map notMap = + new ImmutableMap.Builder() + .put(LITERAL_TRUE, LITERAL_FALSE) + .put(LITERAL_FALSE, LITERAL_TRUE) + .put(LITERAL_NULL, LITERAL_NULL) + .put(LITERAL_MISSING, LITERAL_MISSING) + .build(); + + private static FunctionResolver not() { + FunctionName functionName = BuiltinFunctionName.NOT.getName(); + return FunctionResolver.builder() + .functionName(functionName) + .functionBundle(new FunctionSignature(functionName, + Arrays.asList(BOOLEAN)), predicateFunction(functionName, BOOLEAN)) + .build(); + } + + private static FunctionBuilder predicateFunction( + FunctionName functionName, + ExprType returnType) { + return arguments -> new FunctionExpression(functionName, arguments) { + @Override + public ExprValue valueOf(Environment env) { + return notMap.get(arguments.get(0).valueOf(env)); + } + + @Override + public ExprType type() { + return returnType; + } + + @Override + public String toString() { + return String.format("%s %s", functionName, arguments.get(0).toString()); + } + }; + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/monitor/AlwaysHealthyMonitor.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/monitor/AlwaysHealthyMonitor.java new file mode 100644 index 0000000000..406884f718 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/monitor/AlwaysHealthyMonitor.java @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.monitor; + +/** + * Always healthy resource monitor. + */ +public class AlwaysHealthyMonitor extends ResourceMonitor { + public static final ResourceMonitor ALWAYS_HEALTHY_MONITOR = + new AlwaysHealthyMonitor(); + + /** + * always healthy. + */ + @Override + public boolean isHealthy() { + return true; + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/monitor/ResourceMonitor.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/monitor/ResourceMonitor.java new file mode 100644 index 0000000000..c14e68cf67 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/monitor/ResourceMonitor.java @@ -0,0 +1,29 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.monitor; + +/** + * The abstract interface of ResourceMonitor. + * When an fault is detected, the circuit breaker is open. + */ +public abstract class ResourceMonitor { + /** + * Is the resource healthy. + * + * @return true for healthy, otherwise false. + */ + public abstract boolean isHealthy(); +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementor.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementor.java new file mode 100644 index 0000000000..090cd0c297 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementor.java @@ -0,0 +1,116 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.planner; + +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalAggregation; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalDedupe; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalEval; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalFilter; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanNodeVisitor; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalProject; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRelation; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRemove; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRename; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalSort; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalValues; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.AggregationOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.DedupeOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.EvalOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.FilterOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.ProjectOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.RemoveOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.RenameOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.SortOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.ValuesOperator; + +/** + * Default implementor for implementing logical to physical translation. "Default" here means all + * logical operator will be translated to correspondent physical operator to pipeline operations + * in post-processing style in memory. + * Different storage can override methods here to optimize default pipelining operator, for example + * a storage has the flexibility to override visitFilter and visitRelation to push down filtering + * operation and return a single physical index scan operator. + * + * @param context type + */ +public class DefaultImplementor extends LogicalPlanNodeVisitor { + + @Override + public PhysicalPlan visitDedupe(LogicalDedupe node, C context) { + return new DedupeOperator( + visitChild(node, context), + node.getDedupeList(), + node.getAllowedDuplication(), + node.getKeepEmpty(), + node.getConsecutive()); + } + + @Override + public PhysicalPlan visitProject(LogicalProject node, C context) { + return new ProjectOperator(visitChild(node, context), node.getProjectList()); + } + + @Override + public PhysicalPlan visitRemove(LogicalRemove node, C context) { + return new RemoveOperator(visitChild(node, context), node.getRemoveList()); + } + + @Override + public PhysicalPlan visitEval(LogicalEval node, C context) { + return new EvalOperator(visitChild(node, context), node.getExpressions()); + } + + @Override + public PhysicalPlan visitSort(LogicalSort node, C context) { + return new SortOperator(visitChild(node, context), node.getCount(), node.getSortList()); + } + + @Override + public PhysicalPlan visitRename(LogicalRename node, C context) { + return new RenameOperator(visitChild(node, context), node.getRenameMap()); + } + + @Override + public PhysicalPlan visitAggregation(LogicalAggregation node, C context) { + return new AggregationOperator( + visitChild(node, context), node.getAggregatorList(), node.getGroupByList()); + } + + @Override + public PhysicalPlan visitFilter(LogicalFilter node, C context) { + return new FilterOperator(visitChild(node, context), node.getCondition()); + } + + @Override + public PhysicalPlan visitValues(LogicalValues node, C context) { + return new ValuesOperator(node.getValues()); + } + + @Override + public PhysicalPlan visitRelation(LogicalRelation node, C context) { + throw new UnsupportedOperationException("Storage engine is responsible for " + + "implementing and optimizing logical plan with relation involved"); + } + + protected PhysicalPlan visitChild(LogicalPlan node, C context) { + // Logical operators visited here must have a single child + return node.getChild().get(0).accept(this, context); + } + +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/PlanNode.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/PlanNode.java new file mode 100644 index 0000000000..49bac8cf46 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/PlanNode.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner; + +import java.util.List; + +/** + * The definition of Plan Node. + */ +public interface PlanNode { + + /** + * Return the child nodes. + * + * @return child nodes. + */ + List getChild(); +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/Planner.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/Planner.java new file mode 100644 index 0000000000..e47232bdb6 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/Planner.java @@ -0,0 +1,78 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.planner; + +import static com.google.common.base.Strings.isNullOrEmpty; + +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanNodeVisitor; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRelation; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; +import com.amazon.opendistroforelasticsearch.sql.storage.Table; +import java.util.List; +import lombok.RequiredArgsConstructor; + +/** + * Planner that plans and chooses the optimal physical plan. + */ +@RequiredArgsConstructor +public class Planner { + + /** + * Storage engine. + */ + private final StorageEngine storageEngine; + + /** + * Generate optimal physical plan for logical plan. If no table involved, + * translate logical plan to physical by default implementor. + * TODO: for now just delegate entire logical plan to storage engine. + * + * @param plan logical plan + * @return optimal physical plan + */ + public PhysicalPlan plan(LogicalPlan plan) { + String tableName = findTableName(plan); + if (isNullOrEmpty(tableName)) { + return plan.accept(new DefaultImplementor<>(), null); + } + + Table table = storageEngine.getTable(tableName); + return table.implement(plan); + } + + private String findTableName(LogicalPlan plan) { + return plan.accept(new LogicalPlanNodeVisitor() { + + @Override + protected String visitNode(LogicalPlan node, Object context) { + List children = node.getChild(); + if (children.isEmpty()) { + return ""; + } + return children.get(0).accept(this, context); + } + + @Override + public String visitRelation(LogicalRelation node, Object context) { + return node.getRelationName(); + } + }, null); + } + +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalAggregation.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalAggregation.java new file mode 100644 index 0000000000..ef89d91cef --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalAggregation.java @@ -0,0 +1,49 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; +import java.util.Collections; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Logical Aggregation. + */ +@ToString +@EqualsAndHashCode +@RequiredArgsConstructor +public class LogicalAggregation extends LogicalPlan { + private final LogicalPlan child; + @Getter + private final List aggregatorList; + @Getter + private final List groupByList; + + @Override + public List getChild() { + return Collections.singletonList(child); + } + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitAggregation(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalDedupe.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalDedupe.java new file mode 100644 index 0000000000..5276c7a0b6 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalDedupe.java @@ -0,0 +1,49 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import java.util.Arrays; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Logical Dedupe Plan. + */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class LogicalDedupe extends LogicalPlan { + private final LogicalPlan child; + private final List dedupeList; + private final Integer allowedDuplication; + private final Boolean keepEmpty; + private final Boolean consecutive; + + @Override + public List getChild() { + return Arrays.asList(child); + } + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitDedupe(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalEval.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalEval.java new file mode 100644 index 0000000000..5e3f117610 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalEval.java @@ -0,0 +1,50 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import java.util.Collections; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; +import org.apache.commons.lang3.tuple.Pair; + +/** + * Logical Evaluation represent the evaluation operation. The {@link LogicalEval#expressions} is a + * list assignment operation. e.g. velocity = distance/speed, then the Pair is (velocity, + * distance/speed). + */ +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class LogicalEval extends LogicalPlan { + private final LogicalPlan child; + @Getter + private final List> expressions; + + @Override + public List getChild() { + return Collections.singletonList(child); + } + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitEval(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalFilter.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalFilter.java new file mode 100644 index 0000000000..306104ca6f --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalFilter.java @@ -0,0 +1,46 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import java.util.Arrays; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Logical Filter represent the filter relation. + */ +@ToString +@EqualsAndHashCode +@RequiredArgsConstructor +public class LogicalFilter extends LogicalPlan { + private final LogicalPlan child; + @Getter + private final Expression condition; + + @Override + public List getChild() { + return Arrays.asList(child); + } + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitFilter(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlan.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlan.java new file mode 100644 index 0000000000..035598d568 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlan.java @@ -0,0 +1,34 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +import com.amazon.opendistroforelasticsearch.sql.planner.PlanNode; + +/** + * The abstract base class for all the Logical Plan node. + */ +public abstract class LogicalPlan implements PlanNode { + /** + * Accept the {@link LogicalPlanNodeVisitor}. + * + * @param visitor visitor. + * @param context visitor context. + * @param returned object type. + * @param context type. + * @return returned object. + */ + public abstract R accept(LogicalPlanNodeVisitor visitor, C context); +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanDSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanDSL.java new file mode 100644 index 0000000000..2f828ad9e6 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanDSL.java @@ -0,0 +1,90 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort.SortOption; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.LiteralExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; +import com.google.common.collect.ImmutableSet; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import lombok.experimental.UtilityClass; +import org.apache.commons.lang3.tuple.Pair; + +/** + * Logical Plan DSL. + */ +@UtilityClass +public class LogicalPlanDSL { + public static LogicalPlan aggregation( + LogicalPlan input, List aggregatorList, List groupByList) { + return new LogicalAggregation(input, aggregatorList, groupByList); + } + + public static LogicalPlan filter(LogicalPlan input, Expression expression) { + return new LogicalFilter(input, expression); + } + + public static LogicalPlan relation(String tableName) { + return new LogicalRelation(tableName); + } + + public static LogicalPlan rename( + LogicalPlan input, Map renameMap) { + return new LogicalRename(input, renameMap); + } + + public static LogicalPlan project(LogicalPlan input, Expression... fields) { + return new LogicalProject(input, Arrays.asList(fields)); + } + + public static LogicalPlan remove(LogicalPlan input, ReferenceExpression... fields) { + return new LogicalRemove(input, ImmutableSet.copyOf(fields)); + } + + public static LogicalPlan eval( + LogicalPlan input, Pair... expressions) { + return new LogicalEval(input, Arrays.asList(expressions)); + } + + public static LogicalPlan sort( + LogicalPlan input, Integer count, Pair... sorts) { + return new LogicalSort(input, count, Arrays.asList(sorts)); + } + + public static LogicalPlan dedupe(LogicalPlan input, Expression... fields) { + return dedupe(input, 1, false, false, fields); + } + + public static LogicalPlan dedupe( + LogicalPlan input, + int allowedDuplication, + boolean keepEmpty, + boolean consecutive, + Expression... fields) { + return new LogicalDedupe( + input, Arrays.asList(fields), allowedDuplication, keepEmpty, consecutive); + } + + @SafeVarargs + public LogicalPlan values(List... values) { + return new LogicalValues(Arrays.asList(values)); + } + +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitor.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitor.java new file mode 100644 index 0000000000..bd63b84e5d --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitor.java @@ -0,0 +1,68 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +/** + * The visitor of {@link LogicalPlan}. + * + * @param return object type. + * @param context type. + */ +public abstract class LogicalPlanNodeVisitor { + protected R visitNode(LogicalPlan plan, C context) { + return null; + } + + public R visitRelation(LogicalRelation plan, C context) { + return visitNode(plan, context); + } + + public R visitFilter(LogicalFilter plan, C context) { + return visitNode(plan, context); + } + + public R visitAggregation(LogicalAggregation plan, C context) { + return visitNode(plan, context); + } + + public R visitDedupe(LogicalDedupe plan, C context) { + return visitNode(plan, context); + } + + public R visitRename(LogicalRename plan, C context) { + return visitNode(plan, context); + } + + public R visitProject(LogicalProject plan, C context) { + return visitNode(plan, context); + } + + public R visitRemove(LogicalRemove plan, C context) { + return visitNode(plan, context); + } + + public R visitEval(LogicalEval plan, C context) { + return visitNode(plan, context); + } + + public R visitSort(LogicalSort plan, C context) { + return visitNode(plan, context); + } + + public R visitValues(LogicalValues plan, C context) { + return visitNode(plan, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalProject.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalProject.java new file mode 100644 index 0000000000..edf179903c --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalProject.java @@ -0,0 +1,46 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import java.util.Arrays; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Project field specified by the {@link LogicalProject#projectList}. + */ +@ToString +@EqualsAndHashCode +@RequiredArgsConstructor +public class LogicalProject extends LogicalPlan { + private final LogicalPlan child; + @Getter + private final List projectList; + + @Override + public List getChild() { + return Arrays.asList(child); + } + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitProject(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRelation.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRelation.java new file mode 100644 index 0000000000..a836ce043b --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRelation.java @@ -0,0 +1,44 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Logical Relation represent the data source. + */ +@ToString +@EqualsAndHashCode +@RequiredArgsConstructor +public class LogicalRelation extends LogicalPlan { + @Getter + private final String relationName; + + @Override + public List getChild() { + return ImmutableList.of(); + } + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitRelation(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRemove.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRemove.java new file mode 100644 index 0000000000..11d280cd9b --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRemove.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Remove field specified by the {@link LogicalRemove#removeList}. + */ +@ToString +@EqualsAndHashCode +@RequiredArgsConstructor +public class LogicalRemove extends LogicalPlan { + private final LogicalPlan child; + @Getter + private final Set removeList; + + @Override + public List getChild() { + return Arrays.asList(child); + } + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitRemove(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRename.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRename.java new file mode 100644 index 0000000000..812ac329e1 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRename.java @@ -0,0 +1,48 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Rename Operator. + * renameList is list of mapping of source and target. + */ +@ToString +@EqualsAndHashCode +@RequiredArgsConstructor +public class LogicalRename extends LogicalPlan { + private final LogicalPlan child; + @Getter + private final Map renameMap; + + @Override + public List getChild() { + return Collections.singletonList(child); + } + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitRename(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalSort.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalSort.java new file mode 100644 index 0000000000..4cebc975d8 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalSort.java @@ -0,0 +1,49 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort.SortOption; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import java.util.Arrays; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; +import org.apache.commons.lang3.tuple.Pair; + +/** + * Sort Plan. + */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class LogicalSort extends LogicalPlan { + private final LogicalPlan child; + private final Integer count; + private final List> sortList; + + @Override + public List getChild() { + return Arrays.asList(child); + } + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitSort(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalValues.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalValues.java new file mode 100644 index 0000000000..8ad790da7d --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalValues.java @@ -0,0 +1,60 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +import com.amazon.opendistroforelasticsearch.sql.expression.LiteralExpression; +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Logical operator which is a sequence of literal rows (like a relation). + * Basically, Values operator is used to create rows of constant literals + * "out of nothing" which is corresponding with VALUES clause in SQL. + * Mostly all rows must have the same number of literals and each column should + * have same type or can be converted implicitly. + * In particular, typical use cases include: + * 1. Project without relation involved. + * 2. Defining query or insertion without a relation. + * Take the following logical plan for example: + * 
+ *  LogicalProject(expr=[log(2),true,1+2])
+ *   |_ LogicalValues([[]])  #an empty row so that Project can evaluate its expressions in next()
+ *
+ */ +@ToString +@Getter +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class LogicalValues extends LogicalPlan { + + private final List> values; + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitValues(this, context); + } + + @Override + public List getChild() { + return ImmutableList.of(); + } + +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/AggregationOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/AggregationOperator.java new file mode 100644 index 0000000000..9fd395e8e3 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/AggregationOperator.java @@ -0,0 +1,177 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTupleValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.AggregationState; +import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; +import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.BindingTuple; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Group the all the input {@link BindingTuple} by {@link AggregationOperator#groupByExprList}, + * calculate the aggregation result by using {@link AggregationOperator#aggregatorList}. + */ +@EqualsAndHashCode +@ToString +public class AggregationOperator extends PhysicalPlan { + @Getter + private final PhysicalPlan input; + @Getter + private final List aggregatorList; + @Getter + private final List groupByExprList; + @EqualsAndHashCode.Exclude + private final Group group; + @EqualsAndHashCode.Exclude + private Iterator iterator; + + /** + * AggregationOperator Constructor. + * @param input Input {@link PhysicalPlan} + * @param aggregatorList List of {@link Aggregator} + * @param groupByExprList List of group by {@link Expression} + */ + public AggregationOperator(PhysicalPlan input, List aggregatorList, + List groupByExprList) { + this.input = input; + this.aggregatorList = aggregatorList; + this.groupByExprList = groupByExprList; + this.group = new Group(); + } + + @Override + public R accept(PhysicalPlanNodeVisitor visitor, C context) { + return visitor.visitAggregation(this, context); + } + + @Override + public List getChild() { + return Collections.singletonList(input); + } + + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public ExprValue next() { + return iterator.next(); + } + + @Override + public void open() { + super.open(); + while (input.hasNext()) { + group.push(input.next()); + } + iterator = group.result().iterator(); + } + + @VisibleForTesting + @RequiredArgsConstructor + public class Group { + + private final Map>> groupListMap = + new HashMap<>(); + + /** + * Push the BindingTuple to Group. Two functions will be applied to each BindingTuple to + * generate the {@link GroupKey} and {@link AggregationState} + * Key = GroupKey(bindingTuple), State = Aggregator(bindingTuple) + */ + public void push(ExprValue inputValue) { + GroupKey groupKey = new GroupKey(inputValue); + groupListMap.computeIfAbsent(groupKey, k -> + aggregatorList.stream() + .map(aggregator -> new AbstractMap.SimpleEntry<>(aggregator, + aggregator.create())) + .collect(Collectors.toList()) + ); + groupListMap.computeIfPresent(groupKey, (key, aggregatorList) -> { + aggregatorList + .forEach(entry -> entry.getKey().iterate(inputValue.bindingTuples(), entry.getValue())); + return aggregatorList; + }); + } + + /** + * Get the list of {@link BindingTuple} for each group. + */ + public List result() { + ImmutableList.Builder resultBuilder = new ImmutableList.Builder<>(); + for (Map.Entry>> entry : groupListMap + .entrySet()) { + LinkedHashMap map = new LinkedHashMap<>(); + map.putAll(entry.getKey().groupKeyMap()); + for (Map.Entry stateEntry : entry.getValue()) { + map.put(stateEntry.getKey().toString(), stateEntry.getValue().result()); + } + resultBuilder.add(ExprTupleValue.fromExprValueMap(map)); + } + return resultBuilder.build(); + } + } + + /** + * Group Key. + */ + @EqualsAndHashCode + @VisibleForTesting + public class GroupKey { + + private final List groupByValueList; + + /** + * GroupKey constructor. + */ + public GroupKey(ExprValue value) { + this.groupByValueList = new ArrayList<>(); + for (Expression groupExpr : groupByExprList) { + this.groupByValueList.add(groupExpr.valueOf(value.bindingTuples())); + } + } + + /** + * Return the Map of group field and group field value. + */ + public LinkedHashMap groupKeyMap() { + LinkedHashMap map = new LinkedHashMap<>(); + for (int i = 0; i < groupByExprList.size(); i++) { + map.put(groupByExprList.get(i).toString(), groupByValueList.get(i)); + } + return map; + } + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/DedupeOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/DedupeOperator.java new file mode 100644 index 0000000000..e8efadd51a --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/DedupeOperator.java @@ -0,0 +1,184 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.BindingTuple; +import com.google.common.collect.ImmutableList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.BiFunction; +import java.util.function.Predicate; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; + +/** + * Dedupe operator. Dedupe the input {@link ExprValue} by using the {@link + * DedupeOperator#dedupeList} The result order follow the input order. + */ +@Getter +@EqualsAndHashCode +public class DedupeOperator extends PhysicalPlan { + @Getter + private final PhysicalPlan input; + @Getter + private final List dedupeList; + @Getter + private final Integer allowedDuplication; + @Getter + private final Boolean keepEmpty; + @Getter + private final Boolean consecutive; + + @EqualsAndHashCode.Exclude + private final Deduper> deduper; + @EqualsAndHashCode.Exclude + private ExprValue next; + + private static final Integer ALL_ONE_DUPLICATION = 1; + private static final Boolean IGNORE_EMPTY = false; + private static final Boolean NON_CONSECUTIVE = false; + private static final Predicate NULL_OR_MISSING = v -> v.isNull() || v.isMissing(); + private static final Integer SEEN_FIRST_TIME = 1; + + @NonNull + public DedupeOperator(PhysicalPlan input, List dedupeList) { + this(input, dedupeList, ALL_ONE_DUPLICATION, IGNORE_EMPTY, NON_CONSECUTIVE); + } + + /** + * Dedup Constructor. + * @param input input {@link PhysicalPlan} + * @param dedupeList list of dedupe {@link Expression} + * @param allowedDuplication max allowed duplication + * @param keepEmpty keep empty + * @param consecutive consecutive mode + */ + @NonNull + public DedupeOperator( + PhysicalPlan input, + List dedupeList, + Integer allowedDuplication, + Boolean keepEmpty, + Boolean consecutive) { + this.input = input; + this.dedupeList = dedupeList; + this.allowedDuplication = allowedDuplication; + this.keepEmpty = keepEmpty; + this.consecutive = consecutive; + this.deduper = this.consecutive ? Deduper.consecutiveDeduper() : Deduper.historicalDeduper(); + } + + @Override + public R accept(PhysicalPlanNodeVisitor visitor, C context) { + return visitor.visitDedupe(this, context); + } + + @Override + public List getChild() { + return Collections.singletonList(input); + } + + @Override + public boolean hasNext() { + while (input.hasNext()) { + ExprValue next = input.next(); + if (keep(next)) { + this.next = next; + return true; + } + } + return false; + } + + @Override + public ExprValue next() { + return this.next; + } + + /** + * Test the {@link ExprValue} should be keep or ignore + * + *

If any value evaluted by {@link DedupeOperator#dedupeList} is NULL or MISSING, then the * + * return value is decided by keepEmpty option, default value is ignore. + * + * @param value {@link ExprValue}. + * @return true: keep, false: ignore + */ + public boolean keep(ExprValue value) { + BindingTuple bindingTuple = value.bindingTuples(); + ImmutableList.Builder dedupeKeyBuilder = new ImmutableList.Builder<>(); + for (Expression expression : dedupeList) { + ExprValue exprValue = expression.valueOf(bindingTuple); + if (NULL_OR_MISSING.test(exprValue)) { + return keepEmpty; + } + dedupeKeyBuilder.add(exprValue); + } + List dedupeKey = dedupeKeyBuilder.build(); + int seenTimes = deduper.seenTimes(dedupeKey); + return seenTimes <= allowedDuplication; + } + + /** + * Return how many times the dedupeKey has been seen before. The side effect is the seen times + * will add 1 times after calling this function. + * + * @param dedupe key + */ + @RequiredArgsConstructor + static class Deduper { + private final BiFunction, K, Integer> seenFirstTime; + private final Map seenMap = new ConcurrentHashMap<>(); + + /** + * The Historical Deduper monitor the duplicated element with all the seen value. + */ + public static Deduper historicalDeduper() { + return new Deduper<>( + (map, key) -> { + map.put(key, SEEN_FIRST_TIME); + return SEEN_FIRST_TIME; + }); + } + + /** + * The Consecutive Deduper monitor the duplicated element with consecutive seen value. It means + * only the consecutive duplicated value will be counted. + */ + public static Deduper consecutiveDeduper() { + return new Deduper<>( + (map, key) -> { + map.clear(); + map.put(key, SEEN_FIRST_TIME); + return SEEN_FIRST_TIME; + }); + } + + public int seenTimes(K dedupeKey) { + if (seenMap.containsKey(dedupeKey)) { + return seenMap.computeIfPresent(dedupeKey, (k, v) -> v + 1); + } else { + return seenFirstTime.apply(seenMap, dedupeKey); + } + } + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/EvalOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/EvalOperator.java new file mode 100644 index 0000000000..6fc9d75655 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/EvalOperator.java @@ -0,0 +1,111 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRUCT; +import static com.amazon.opendistroforelasticsearch.sql.expression.env.Environment.extendEnv; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTupleValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap.Builder; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; +import org.apache.commons.lang3.tuple.Pair; + +/** + * The eval operator evaluate the {@link EvalOperator#expressionList} and put the result into to + * output. If the field name doesn't exist in the input, a new field will be append to the output. + * If the field name exist in the input, a new value will be put into to output. + * + *

The {@link EvalOperator#expressionList} are evaluated from left to right. It means you can + * reference previous evaluated field. + * e.g. fields velocity = distance/time, doubleVelocity = 2 * velocity + */ +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class EvalOperator extends PhysicalPlan { + @Getter + private final PhysicalPlan input; + @Getter + private final List> expressionList; + + @Override + public R accept(PhysicalPlanNodeVisitor visitor, C context) { + return visitor.visitEval(this, context); + } + + @Override + public List getChild() { + return Collections.singletonList(input); + } + + @Override + public boolean hasNext() { + return input.hasNext(); + } + + @Override + public ExprValue next() { + ExprValue inputValue = input.next(); + Map evalMap = eval(inputValue.bindingTuples()); + + if (STRUCT == inputValue.type()) { + ImmutableMap.Builder resultBuilder = new Builder<>(); + Map tupleValue = ExprValueUtils.getTupleValue(inputValue); + for (Entry valueEntry : tupleValue.entrySet()) { + if (evalMap.containsKey(valueEntry.getKey())) { + resultBuilder.put(valueEntry.getKey(), evalMap.get(valueEntry.getKey())); + evalMap.remove(valueEntry.getKey()); + } else { + resultBuilder.put(valueEntry); + } + } + resultBuilder.putAll(evalMap); + return ExprTupleValue.fromExprValueMap(resultBuilder.build()); + } else { + return inputValue; + } + } + + /** + * Evaluate the expression in the {@link EvalOperator#expressionList} with {@link Environment}. + * @param env {@link Environment} + * @return The mapping of reference and {@link ExprValue} for each expression. + */ + private Map eval(Environment env) { + Map evalResultMap = new LinkedHashMap<>(); + for (Pair pair : expressionList) { + ReferenceExpression var = pair.getKey(); + ExprValue value = pair.getValue().valueOf(env); + env = extendEnv(env, var, value); + evalResultMap.put(var.toString(), value); + } + return evalResultMap; + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/FilterOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/FilterOperator.java new file mode 100644 index 0000000000..be6d48be39 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/FilterOperator.java @@ -0,0 +1,57 @@ +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.operator.predicate.BinaryPredicateOperator; +import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.BindingTuple; +import java.util.Collections; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * The Filter operator use the conditions to evaluate the input {@link BindingTuple}. + * The Filter operator only return the results that evaluated to true. + * The NULL and MISSING are handled by the logic defined in {@link BinaryPredicateOperator}. + */ +@EqualsAndHashCode +@ToString +@RequiredArgsConstructor +public class FilterOperator extends PhysicalPlan { + @Getter + private final PhysicalPlan input; + @Getter + private final Expression conditions; + @ToString.Exclude private ExprValue next = null; + + @Override + public R accept(PhysicalPlanNodeVisitor visitor, C context) { + return visitor.visitFilter(this, context); + } + + @Override + public List getChild() { + return Collections.singletonList(input); + } + + @Override + public boolean hasNext() { + while (input.hasNext()) { + ExprValue inputValue = input.next(); + ExprValue exprValue = conditions.valueOf(inputValue.bindingTuples()); + if (ExprValueUtils.getBooleanValue(exprValue)) { + next = inputValue; + return true; + } + } + return false; + } + + @Override + public ExprValue next() { + return next; + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlan.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlan.java new file mode 100644 index 0000000000..e7f64c84ec --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlan.java @@ -0,0 +1,46 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.planner.PlanNode; +import java.util.Iterator; + +/** + * Physical plan. + */ +public abstract class PhysicalPlan implements PlanNode, + Iterator, + AutoCloseable { + /** + * Accept the {@link PhysicalPlanNodeVisitor}. + * + * @param visitor visitor. + * @param context visitor context. + * @param returned object type. + * @param context type. + * @return returned object. + */ + public abstract R accept(PhysicalPlanNodeVisitor visitor, C context); + + public void open() { + getChild().forEach(PhysicalPlan::open); + } + + public void close() { + getChild().forEach(PhysicalPlan::close); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanDSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanDSL.java new file mode 100644 index 0000000000..6dedd39e04 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanDSL.java @@ -0,0 +1,87 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort.SortOption; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.LiteralExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; +import com.google.common.collect.ImmutableSet; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import lombok.experimental.UtilityClass; +import org.apache.commons.lang3.tuple.Pair; + +/** + * Physical Plan DSL. + */ +@UtilityClass +public class PhysicalPlanDSL { + + public static AggregationOperator agg( + PhysicalPlan input, List aggregators, List groups) { + return new AggregationOperator(input, aggregators, groups); + } + + public static FilterOperator filter(PhysicalPlan input, Expression condition) { + return new FilterOperator(input, condition); + } + + public static RenameOperator rename( + PhysicalPlan input, Map renameMap) { + return new RenameOperator(input, renameMap); + } + + public static ProjectOperator project(PhysicalPlan input, Expression... fields) { + return new ProjectOperator(input, Arrays.asList(fields)); + } + + public static RemoveOperator remove(PhysicalPlan input, ReferenceExpression... fields) { + return new RemoveOperator(input, ImmutableSet.copyOf(fields)); + } + + public static EvalOperator eval( + PhysicalPlan input, Pair... expressions) { + return new EvalOperator(input, Arrays.asList(expressions)); + } + + public static SortOperator sort(PhysicalPlan input, Integer count, Pair... sorts) { + return new SortOperator(input, count, Arrays.asList(sorts)); + } + + public static DedupeOperator dedupe(PhysicalPlan input, Expression... expressions) { + return new DedupeOperator(input, Arrays.asList(expressions)); + } + + public static DedupeOperator dedupe( + PhysicalPlan input, + int allowedDuplication, + boolean keepEmpty, + boolean consecutive, + Expression... expressions) { + return new DedupeOperator( + input, Arrays.asList(expressions), allowedDuplication, keepEmpty, consecutive); + } + + @SafeVarargs + public ValuesOperator values(List... values) { + return new ValuesOperator(Arrays.asList(values)); + } + +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitor.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitor.java new file mode 100644 index 0000000000..9756b57cbb --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitor.java @@ -0,0 +1,71 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import com.amazon.opendistroforelasticsearch.sql.storage.TableScanOperator; + +/** + * The visitor of {@link PhysicalPlan}. + * + * @param return object type. + * @param context type. + */ +public abstract class PhysicalPlanNodeVisitor { + + protected R visitNode(PhysicalPlan node, C context) { + return null; + } + + public R visitFilter(FilterOperator node, C context) { + return visitNode(node, context); + } + + public R visitAggregation(AggregationOperator node, C context) { + return visitNode(node, context); + } + + public R visitRename(RenameOperator node, C context) { + return visitNode(node, context); + } + + public R visitTableScan(TableScanOperator node, C context) { + return visitNode(node, context); + } + + public R visitProject(ProjectOperator node, C context) { + return visitNode(node, context); + } + + public R visitRemove(RemoveOperator node, C context) { + return visitNode(node, context); + } + + public R visitEval(EvalOperator node, C context) { + return visitNode(node, context); + } + + public R visitDedupe(DedupeOperator node, C context) { + return visitNode(node, context); + } + + public R visitValues(ValuesOperator node, C context) { + return visitNode(node, context); + } + + public R visitSort(SortOperator node, C context) { + return visitNode(node, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperator.java new file mode 100644 index 0000000000..04aa049e57 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperator.java @@ -0,0 +1,70 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTupleValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap.Builder; +import java.util.Collections; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Project the fields specified in {@link ProjectOperator#projectList} from input. + */ +@ToString +@EqualsAndHashCode +@RequiredArgsConstructor +public class ProjectOperator extends PhysicalPlan { + @Getter + private final PhysicalPlan input; + @Getter + private final List projectList; + + @Override + public R accept(PhysicalPlanNodeVisitor visitor, C context) { + return visitor.visitProject(this, context); + } + + @Override + public List getChild() { + return Collections.singletonList(input); + } + + @Override + public boolean hasNext() { + return input.hasNext(); + } + + @Override + public ExprValue next() { + ExprValue inputValue = input.next(); + ImmutableMap.Builder mapBuilder = new Builder<>(); + for (Expression expr : projectList) { + ExprValue exprValue = expr.valueOf(inputValue.bindingTuples()); + // missing value is ignored. + if (!exprValue.isMissing()) { + mapBuilder.put(expr.toString(), exprValue); + } + } + return ExprTupleValue.fromExprValueMap(mapBuilder.build()); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RemoveOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RemoveOperator.java new file mode 100644 index 0000000000..caea9286ff --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RemoveOperator.java @@ -0,0 +1,93 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRUCT; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTupleValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap.Builder; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; + +/** + * Remove the fields specified in {@link RemoveOperator#removeList} from input. + */ +@ToString +@EqualsAndHashCode +public class RemoveOperator extends PhysicalPlan { + @Getter + private final PhysicalPlan input; + @Getter + private final Set removeList; + @ToString.Exclude + @EqualsAndHashCode.Exclude + private final Set nameRemoveList; + + /** + * Todo. This is the temporary solution that add the mapping between string and ref. because when + * rename the field from input, there we can only get the string field. + */ + public RemoveOperator(PhysicalPlan input, + Set removeList) { + this.input = input; + this.removeList = removeList; + this.nameRemoveList = + removeList.stream().map(ReferenceExpression::getAttr).collect(Collectors.toSet()); + } + + @Override + public R accept(PhysicalPlanNodeVisitor visitor, C context) { + return visitor.visitRemove(this, context); + } + + @Override + public List getChild() { + return Collections.singletonList(input); + } + + @Override + public boolean hasNext() { + return input.hasNext(); + } + + @Override + public ExprValue next() { + ExprValue inputValue = input.next(); + if (STRUCT == inputValue.type()) { + ImmutableMap.Builder mapBuilder = new Builder<>(); + Map tupleValue = ExprValueUtils.getTupleValue(inputValue); + for (Entry valueEntry : tupleValue.entrySet()) { + if (!nameRemoveList.contains(valueEntry.getKey())) { + mapBuilder.put(valueEntry); + } + } + return ExprTupleValue.fromExprValueMap(mapBuilder.build()); + } else { + return inputValue; + } + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RenameOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RenameOperator.java new file mode 100644 index 0000000000..d2f55fe814 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RenameOperator.java @@ -0,0 +1,100 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRUCT; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTupleValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.BindingTuple; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap.Builder; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; + +/** + * Rename the binding name in {@link BindingTuple}. + * The mapping maintain the relation between source and target. + * it means BindingTuple.resolve(target) = BindingTuple.resolve(source). + */ +@EqualsAndHashCode +@ToString +public class RenameOperator extends PhysicalPlan { + @Getter + private final PhysicalPlan input; + @Getter + private final Map mapping; + /** + * Todo. This is the temporary solution that add the mapping between string and ref. because when + * rename the field from input, there we can only get the string field. + */ + @ToString.Exclude + @EqualsAndHashCode.Exclude + private final Map nameMapping; + + /** + * Constructor of RenameOperator. + */ + public RenameOperator(PhysicalPlan input, + Map mapping) { + this.input = input; + this.mapping = mapping; + this.nameMapping = + mapping.entrySet().stream().collect(Collectors.toMap(entry -> entry.getKey().getAttr(), + entry -> entry.getValue())); + } + + @Override + public R accept(PhysicalPlanNodeVisitor visitor, C context) { + return visitor.visitRename(this, context); + } + + @Override + public List getChild() { + return Collections.singletonList(input); + } + + @Override + public boolean hasNext() { + return input.hasNext(); + } + + @Override + public ExprValue next() { + ExprValue inputValue = input.next(); + if (STRUCT == inputValue.type()) { + Map tupleValue = ExprValueUtils.getTupleValue(inputValue); + ImmutableMap.Builder mapBuilder = new Builder<>(); + for (String bindName : tupleValue.keySet()) { + if (nameMapping.containsKey(bindName)) { + mapBuilder.put(nameMapping.get(bindName).getAttr(), tupleValue.get(bindName)); + } else { + mapBuilder.put(bindName, tupleValue.get(bindName)); + } + } + return ExprTupleValue.fromExprValueMap(mapBuilder.build()); + } else { + return inputValue; + } + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/SortOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/SortOperator.java new file mode 100644 index 0000000000..3cc1ee163c --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/SortOperator.java @@ -0,0 +1,145 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import static com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort.NullOrder.NULL_FIRST; +import static com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort.SortOrder.ASC; + +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort.SortOption; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.utils.ExprValueOrdering; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.SortOperator.Sorter.SorterBuilder; +import com.google.common.collect.Iterators; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.PriorityQueue; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Singular; +import lombok.ToString; +import org.apache.commons.lang3.tuple.Pair; + +/** + * Sort Operator. The input data is sorted by the sort fields in the {@link SortOperator#sortList}. + * The sort field is specified by the {@link Expression} with {@link SortOption}. The count indicate + * how many sorted result should been return. + */ +@ToString +@EqualsAndHashCode +public class SortOperator extends PhysicalPlan { + @Getter + private final PhysicalPlan input; + @Getter + private final Integer count; + @Getter + private final List> sortList; + @EqualsAndHashCode.Exclude + private final Sorter sorter; + @EqualsAndHashCode.Exclude + private Iterator iterator; + + /** + * Sort Operator Constructor. + * @param input input {@link PhysicalPlan} + * @param count how many sorted result should been return + * @param sortList list of sort sort field. + * The sort field is specified by the {@link Expression} with {@link SortOption} + */ + public SortOperator( + PhysicalPlan input, Integer count, List> sortList) { + this.input = input; + this.count = count; + this.sortList = sortList; + SorterBuilder sorterBuilder = Sorter.builder(); + for (Pair pair : sortList) { + SortOption option = pair.getLeft(); + ExprValueOrdering ordering = + ASC.equals(option.getSortOrder()) + ? ExprValueOrdering.natural() + : ExprValueOrdering.natural().reverse(); + ordering = + NULL_FIRST.equals(option.getNullOrder()) ? ordering.nullsFirst() : ordering.nullsLast(); + sorterBuilder.comparator(Pair.of(pair.getRight(), ordering)); + } + this.sorter = sorterBuilder.build(); + } + + @Override + public R accept(PhysicalPlanNodeVisitor visitor, C context) { + return visitor.visitSort(this, context); + } + + @Override + public void open() { + super.open(); + PriorityQueue sorted = new PriorityQueue<>(1, sorter::compare); + while (input.hasNext()) { + sorted.add(input.next()); + } + iterator = Iterators.limit(new Iterator() { + @Override + public boolean hasNext() { + return !sorted.isEmpty(); + } + + @Override + public ExprValue next() { + return sorted.poll(); + } + }, count); + } + + @Override + public List getChild() { + return Collections.singletonList(input); + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public ExprValue next() { + return iterator.next(); + } + + @Builder + public static class Sorter implements Comparator { + @Singular + private final List>> comparators; + + @Override + public int compare(ExprValue o1, ExprValue o2) { + for (Pair> comparator : comparators) { + Expression expression = comparator.getKey(); + int result = + comparator + .getValue() + .compare( + expression.valueOf(o1.bindingTuples()), expression.valueOf(o2.bindingTuples())); + if (result != 0) { + return result; + } + } + return 0; + } + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ValuesOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ValuesOperator.java new file mode 100644 index 0000000000..215589e5ef --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ValuesOperator.java @@ -0,0 +1,74 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprCollectionValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.expression.LiteralExpression; +import com.google.common.collect.ImmutableList; +import java.util.Iterator; +import java.util.List; +import java.util.stream.Collectors; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +/** + * Physical operator for Values. + */ +@ToString +@EqualsAndHashCode(callSuper = false, of = "values") +public class ValuesOperator extends PhysicalPlan { + + /** + * Original values list for print and equality check. + */ + private final List> values; + + /** + * Values iterator. + */ + private final Iterator> valuesIterator; + + public ValuesOperator(List> values) { + this.values = values; + this.valuesIterator = values.iterator(); + } + + @Override + public R accept(PhysicalPlanNodeVisitor visitor, C context) { + return visitor.visitValues(this, context); + } + + @Override + public List getChild() { + return ImmutableList.of(); + } + + @Override + public boolean hasNext() { + return valuesIterator.hasNext(); + } + + @Override + public ExprValue next() { + List values = valuesIterator.next().stream() + .map(expr -> expr.valueOf(null)) + .collect(Collectors.toList()); + return new ExprCollectionValue(values); + } + +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/storage/StorageEngine.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/storage/StorageEngine.java new file mode 100644 index 0000000000..a9eb5fee7e --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/storage/StorageEngine.java @@ -0,0 +1,27 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.storage; + +/** + * Storage engine for different storage to provide data access API implementation. + */ +public interface StorageEngine { + + /** + * Get {@link Table} from storage engine. + */ + Table getTable(String name); +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/storage/Table.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/storage/Table.java new file mode 100644 index 0000000000..6a17fed62f --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/storage/Table.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.storage; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import java.util.Map; + +/** + * Table. + */ +public interface Table { + + /** + * Get the {@link ExprType} for each field in the table. + */ + Map getFieldTypes(); + + /** + * Implement a {@link LogicalPlan} by {@link PhysicalPlan} in storage engine. + * + * @param plan logical plan + * @return physical plan + */ + PhysicalPlan implement(LogicalPlan plan); + +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/storage/TableScanOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/storage/TableScanOperator.java new file mode 100644 index 0000000000..bcc5a8cab2 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/storage/TableScanOperator.java @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.storage; + +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlanNodeVisitor; +import java.util.Collections; +import java.util.List; + +/** + * Abstract table scan class for different storage to implement. + * This is also to avoid "polluting" physical plan visitor by concrete table scan implementation. + */ +public abstract class TableScanOperator extends PhysicalPlan { + + @Override + public R accept(PhysicalPlanNodeVisitor visitor, C context) { + return visitor.visitTableScan(this, context); + } + + @Override + public List getChild() { + return Collections.emptyList(); + } + +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/storage/bindingtuple/BindingTuple.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/storage/bindingtuple/BindingTuple.java new file mode 100644 index 0000000000..3d1d0aba74 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/storage/bindingtuple/BindingTuple.java @@ -0,0 +1,53 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprMissingValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment; + +/** + * BindingTuple represents the a relationship between bindingName and ExprValue. + * e.g. The operation output column name is bindingName, the value is the ExprValue. + */ +public abstract class BindingTuple implements Environment { + public static BindingTuple EMPTY = new BindingTuple() { + @Override + public ExprValue resolve(ReferenceExpression ref) { + return ExprMissingValue.of(); + } + }; + + /** + * Resolve {@link Expression} in the BindingTuple environment. + */ + @Override + public ExprValue resolve(Expression var) { + if (var instanceof ReferenceExpression) { + return resolve(((ReferenceExpression) var)); + } else { + throw new ExpressionEvaluationException(String.format("can resolve expression: %s", var)); + } + } + + /** + * Resolve the {@link ReferenceExpression} in BindingTuple context. + */ + public abstract ExprValue resolve(ReferenceExpression ref); +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/storage/bindingtuple/LazyBindingTuple.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/storage/bindingtuple/LazyBindingTuple.java new file mode 100644 index 0000000000..f5e29594b9 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/storage/bindingtuple/LazyBindingTuple.java @@ -0,0 +1,34 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import java.util.function.Function; +import lombok.RequiredArgsConstructor; + +/** + * Lazy Implementation of {@link BindingTuple}. + */ +@RequiredArgsConstructor +public class LazyBindingTuple extends BindingTuple { + private final Function lazyBinding; + + @Override + public ExprValue resolve(ReferenceExpression ref) { + return lazyBinding.apply(ref.getAttr()); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/ExpressionUtils.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/ExpressionUtils.java new file mode 100644 index 0000000000..dcb8b10239 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/ExpressionUtils.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.utils; + +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import java.util.List; +import java.util.stream.Collectors; +import lombok.experimental.UtilityClass; + +/** + * Utils for {@link Expression}. + */ +@UtilityClass +public class ExpressionUtils { + + /** + * Format the list of {@link Expression}. + */ + public static String format(List expressionList) { + return expressionList.stream().map(Expression::toString).collect(Collectors.joining(",")); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/OperatorUtils.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/OperatorUtils.java new file mode 100644 index 0000000000..2b5479ebf9 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/OperatorUtils.java @@ -0,0 +1,87 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.utils; + +import java.util.regex.Pattern; +import lombok.experimental.UtilityClass; + +@UtilityClass +public class OperatorUtils { + /** + * Wildcard pattern matcher util. + * Percent (%) character for wildcard, + * Underscore (_) character for a single character match. + * @param pattern string pattern to match. + * @return if text matches pattern returns true; else return false. + */ + public static boolean matches(String pattern, String text) { + return Pattern.compile(patternToRegex(pattern)).matcher(text).matches(); + } + + private static final char DEFAULT_ESCAPE = '\\'; + + private static String patternToRegex(String patternString) { + StringBuilder regex = new StringBuilder(patternString.length() * 2); + regex.append('^'); + boolean escaped = false; + for (char currentChar : patternString.toCharArray()) { + if (!escaped && currentChar == DEFAULT_ESCAPE) { + escaped = true; + } else { + switch (currentChar) { + case '%': + if (escaped) { + regex.append("%"); + } else { + regex.append(".*"); + } + escaped = false; + break; + case '_': + if (escaped) { + regex.append("_"); + } else { + regex.append('.'); + } + escaped = false; + break; + default: + switch (currentChar) { + case '\\': + case '^': + case '$': + case '.': + case '*': + case '[': + case ']': + case '(': + case ')': + case '|': + case '+': + regex.append('\\'); + break; + default: + } + + regex.append(currentChar); + escaped = false; + } + } + } + regex.append('$'); + return regex.toString(); + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalysisContextTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalysisContextTest.java new file mode 100644 index 0000000000..225d962185 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalysisContextTest.java @@ -0,0 +1,50 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.analysis; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +class AnalysisContextTest { + + private final AnalysisContext context = new AnalysisContext(); + + @Test + public void rootEnvironmentShouldBeThereInitially() { + assertNotNull(context.peek()); + } + + @Test + public void pushAndPopEnvironmentShouldPass() { + context.push(); + context.pop(); + } + + @Test + public void popRootEnvironmentShouldPass() { + context.pop(); + } + + @Test + public void popEmptyEnvironmentStackShouldFail() { + context.pop(); + NullPointerException exception = assertThrows(NullPointerException.class, () -> context.pop()); + assertEquals("Fail to pop context due to no environment present", exception.getMessage()); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTest.java new file mode 100644 index 0000000000..b7ffc28387 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTest.java @@ -0,0 +1,192 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.analysis; + +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.argument; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.booleanLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.compare; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.field; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.filter; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.intLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.relation; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.integerValue; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL; +import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.util.Collections; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +class AnalyzerTest extends AnalyzerTestBase { + @Test + public void filter_relation() { + assertAnalyzeEqual( + LogicalPlanDSL.filter( + LogicalPlanDSL.relation("schema"), + dsl.equal(DSL.ref("integer_value", INTEGER), DSL.literal(integerValue(1)))), + AstDSL.filter( + AstDSL.relation("schema"), + AstDSL.equalTo(AstDSL.field("integer_value"), AstDSL.intLiteral(1)))); + } + + @Test + public void analyze_filter_relation() { + assertAnalyzeEqual( + LogicalPlanDSL.filter( + LogicalPlanDSL.relation("schema"), + dsl.equal(DSL.ref("integer_value", INTEGER), DSL.literal(integerValue(1)))), + filter(relation("schema"), compare("=", field("integer_value"), intLiteral(1)))); + } + + @Test + public void rename_relation() { + assertAnalyzeEqual( + LogicalPlanDSL.rename( + LogicalPlanDSL.relation("schema"), + ImmutableMap.of(DSL.ref("integer_value", INTEGER), DSL.ref("ivalue", INTEGER))), + AstDSL.rename( + AstDSL.relation("schema"), + AstDSL.map(AstDSL.field("integer_value"), AstDSL.field("ivalue")))); + } + + @Test + public void rename_stats_source() { + assertAnalyzeEqual( + LogicalPlanDSL.rename( + LogicalPlanDSL.aggregation( + LogicalPlanDSL.relation("schema"), + ImmutableList.of(dsl.avg(DSL.ref("integer_value", INTEGER))), + ImmutableList.of()), + ImmutableMap.of(DSL.ref("avg(integer_value)", DOUBLE), DSL.ref("ivalue", DOUBLE))), + AstDSL.rename( + AstDSL.agg( + AstDSL.relation("schema"), + AstDSL.exprList(AstDSL.aggregate("avg", field("integer_value"))), + null, + ImmutableList.of(), + AstDSL.defaultStatsArgs()), + AstDSL.map(AstDSL.aggregate("avg", field("integer_value")), field("ivalue")))); + } + + @Test + public void stats_source() { + assertAnalyzeEqual( + LogicalPlanDSL.aggregation( + LogicalPlanDSL.relation("schema"), + ImmutableList.of(dsl.avg(DSL.ref("integer_value", INTEGER))), + ImmutableList.of(DSL.ref("string_value", STRING))), + AstDSL.agg( + AstDSL.relation("schema"), + AstDSL.exprList(AstDSL.aggregate("avg", field("integer_value"))), + null, + ImmutableList.of(field("string_value")), + AstDSL.defaultStatsArgs())); + } + + @Test + public void rename_to_invalid_expression() { + SemanticCheckException exception = + assertThrows( + SemanticCheckException.class, + () -> + analyze( + AstDSL.rename( + AstDSL.agg( + AstDSL.relation("schema"), + AstDSL.exprList(AstDSL.aggregate("avg", field("integer_value"))), + Collections.emptyList(), + ImmutableList.of(), + AstDSL.defaultStatsArgs()), + AstDSL.map( + AstDSL.aggregate("avg", field("integer_value")), + AstDSL.aggregate("avg", field("integer_value")))))); + assertEquals( + "the target expected to be field, but is avg(Field(field=integer_value, fieldArgs=[]))", + exception.getMessage()); + } + + @Test + public void project_source() { + assertAnalyzeEqual( + LogicalPlanDSL.project( + LogicalPlanDSL.relation("schema"), DSL.ref("integer_value", INTEGER), DSL.ref( + "double_value", DOUBLE)), + AstDSL.projectWithArg( + AstDSL.relation("schema"), + AstDSL.defaultFieldsArgs(), + AstDSL.field("integer_value"), + AstDSL.field("double_value"))); + } + + @Test + public void remove_source() { + assertAnalyzeEqual( + LogicalPlanDSL.remove( + LogicalPlanDSL.relation("schema"), DSL.ref("integer_value", INTEGER), DSL.ref( + "double_value", DOUBLE)), + AstDSL.projectWithArg( + AstDSL.relation("schema"), + Collections.singletonList(argument("exclude", booleanLiteral(true))), + AstDSL.field("integer_value"), + AstDSL.field("double_value"))); + } + + @Disabled("the project/remove command should shrink the type env") + @Test + public void project_source_change_type_env() { + SemanticCheckException exception = + assertThrows( + SemanticCheckException.class, + () -> + analyze( + AstDSL.projectWithArg( + AstDSL.projectWithArg( + AstDSL.relation("schema"), + AstDSL.defaultFieldsArgs(), + AstDSL.field("integer_value"), + AstDSL.field("double_value")), + AstDSL.defaultFieldsArgs(), + AstDSL.field("float_value")))); + } + + @Test + public void project_values() { + assertAnalyzeEqual( + LogicalPlanDSL.project( + LogicalPlanDSL.values(ImmutableList.of(DSL.literal(123))), + DSL.literal(123), + DSL.literal("hello"), + DSL.literal(false) + ), + AstDSL.project( + AstDSL.values(ImmutableList.of(AstDSL.intLiteral(123))), + AstDSL.intLiteral(123), + AstDSL.stringLiteral("hello"), + AstDSL.booleanLiteral(false) + ) + ); + } + +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTestBase.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTestBase.java new file mode 100644 index 0000000000..18e23cf1ba --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTestBase.java @@ -0,0 +1,85 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.analysis; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.amazon.opendistroforelasticsearch.sql.analysis.symbol.SymbolTable; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; +import com.amazon.opendistroforelasticsearch.sql.config.TestConfig; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.config.ExpressionConfig; +import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +@Configuration +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {ExpressionConfig.class, AnalyzerTestBase.class, TestConfig.class}) +public class AnalyzerTestBase { + + @Autowired + protected DSL dsl; + + @Autowired + protected AnalysisContext analysisContext; + + @Autowired + protected ExpressionAnalyzer expressionAnalyzer; + + @Autowired + protected Analyzer analyzer; + + @Autowired + protected Environment typeEnv; + + @Bean + protected Analyzer analyzer(ExpressionAnalyzer expressionAnalyzer, StorageEngine engine) { + return new Analyzer(expressionAnalyzer, engine); + } + + @Bean + protected TypeEnvironment typeEnvironment(SymbolTable symbolTable) { + return new TypeEnvironment(null, symbolTable); + } + + @Bean + protected AnalysisContext analysisContext(TypeEnvironment typeEnvironment) { + return new AnalysisContext(typeEnvironment); + } + + @Bean + protected ExpressionAnalyzer expressionAnalyzer(DSL dsl, BuiltinFunctionRepository repo) { + return new ExpressionAnalyzer(repo); + } + + protected void assertAnalyzeEqual(LogicalPlan expected, UnresolvedPlan unresolvedPlan) { + assertEquals(expected, analyze(unresolvedPlan)); + } + + protected LogicalPlan analyze(UnresolvedPlan unresolvedPlan) { + return analyzer.analyze(unresolvedPlan, analysisContext); + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/ExpressionAnalyzerTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/ExpressionAnalyzerTest.java new file mode 100644 index 0000000000..b9ff216110 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/ExpressionAnalyzerTest.java @@ -0,0 +1,100 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.analysis; + +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.field; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.integerValue; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BOOLEAN; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; +import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import org.junit.jupiter.api.Test; + + +class ExpressionAnalyzerTest extends AnalyzerTestBase { + + @Test + public void equal() { + assertAnalyzeEqual( + dsl.equal(DSL.ref("integer_value", INTEGER), DSL.literal(integerValue(1))), + AstDSL.equalTo(AstDSL.unresolvedAttr("integer_value"), AstDSL.intLiteral(1)) + ); + } + + @Test + public void and() { + assertAnalyzeEqual( + dsl.and(DSL.ref("boolean_value", BOOLEAN), DSL.literal(LITERAL_TRUE)), + AstDSL.and(AstDSL.unresolvedAttr("boolean_value"), AstDSL.booleanLiteral(true)) + ); + } + + @Test + public void or() { + assertAnalyzeEqual( + dsl.or(DSL.ref("boolean_value", BOOLEAN), DSL.literal(LITERAL_TRUE)), + AstDSL.or(AstDSL.unresolvedAttr("boolean_value"), AstDSL.booleanLiteral(true)) + ); + } + + @Test + public void xor() { + assertAnalyzeEqual( + dsl.xor(DSL.ref("boolean_value", BOOLEAN), DSL.literal(LITERAL_TRUE)), + AstDSL.xor(AstDSL.unresolvedAttr("boolean_value"), AstDSL.booleanLiteral(true)) + ); + } + + @Test + public void not() { + assertAnalyzeEqual( + dsl.not(DSL.ref("boolean_value", BOOLEAN)), + AstDSL.not(AstDSL.unresolvedAttr("boolean_value")) + ); + } + + @Test + public void undefined_var_semantic_check_failed() { + SemanticCheckException exception = assertThrows(SemanticCheckException.class, + () -> analyze( + AstDSL.and(AstDSL.unresolvedAttr("undefined_field"), AstDSL.booleanLiteral(true)))); + assertEquals("can't resolve Symbol(namespace=FIELD_NAME, name=undefined_field) in type env", + exception.getMessage()); + } + + @Test + public void undefined_aggregation_function() { + SemanticCheckException exception = assertThrows(SemanticCheckException.class, + () -> analyze(AstDSL.aggregate("ESTDC_ERROR", field("integer_value")))); + assertEquals("Unsupported aggregation function ESTDC_ERROR", exception.getMessage()); + } + + protected Expression analyze(UnresolvedExpression unresolvedExpression) { + return expressionAnalyzer.analyze(unresolvedExpression, analysisContext); + } + + protected void assertAnalyzeEqual(Expression expected, + UnresolvedExpression unresolvedExpression) { + assertEquals(expected, analyze(unresolvedExpression)); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/TypeEnvironmentTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/TypeEnvironmentTest.java new file mode 100644 index 0000000000..f038b671d5 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/TypeEnvironmentTest.java @@ -0,0 +1,105 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.analysis; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRUCT; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.amazon.opendistroforelasticsearch.sql.analysis.symbol.Namespace; +import com.amazon.opendistroforelasticsearch.sql.analysis.symbol.Symbol; +import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import org.junit.jupiter.api.Test; + +public class TypeEnvironmentTest { + + /** + * Use context class for push/pop. + */ + private AnalysisContext context = new AnalysisContext(); + + @Test + public void defineFieldSymbolInDifferentEnvironmentsShouldBeAbleToResolve() { + // Root environment + ReferenceExpression age = DSL.ref("s.age", INTEGER); + environment().define(age); + assertEquals(INTEGER, environment().resolve(toSymbol(age))); + + // New environment 1 + context.push(); + ReferenceExpression city = DSL.ref("s.city", STRING); + environment().define(city); + assertEquals(INTEGER, environment().resolve(toSymbol(age))); + assertEquals(STRING, environment().resolve(toSymbol(city))); + + // New environment 2 + context.push(); + ReferenceExpression manager = DSL.ref("s.manager", STRUCT); + environment().define(manager); + assertEquals(INTEGER, environment().resolve(toSymbol(age))); + assertEquals(STRING, environment().resolve(toSymbol(city))); + assertEquals(STRUCT, environment().resolve(toSymbol(manager))); + } + + @Test + public void defineFieldSymbolInDifferentEnvironmentsShouldNotAbleToResolveOncePopped() { + // Root environment + ReferenceExpression age = DSL.ref("s.age", INTEGER); + environment().define(age); + + // New environment + context.push(); + ReferenceExpression city = DSL.ref("s.city", STRING); + environment().define(city); + ReferenceExpression manager = DSL.ref("s.manager", STRUCT); + environment().define(manager); + assertEquals(INTEGER, environment().resolve(toSymbol(age))); + assertEquals(STRING, environment().resolve(toSymbol(city))); + assertEquals(STRUCT, environment().resolve(toSymbol(manager))); + + context.pop(); + assertEquals(INTEGER, environment().resolve(toSymbol(age))); + SemanticCheckException exception = + assertThrows(SemanticCheckException.class, () -> environment().resolve(toSymbol(city))); + assertEquals("can't resolve Symbol(namespace=FIELD_NAME, name=s.city) in type env", + exception.getMessage()); + exception = assertThrows(SemanticCheckException.class, + () -> environment().resolve(toSymbol(manager))); + assertEquals("can't resolve Symbol(namespace=FIELD_NAME, name=s.manager) in type env", + exception.getMessage()); + } + + @Test + public void resolveLiteralInEnvFailed() { + SemanticCheckException exception = assertThrows(SemanticCheckException.class, + () -> environment().resolve(new Symbol(Namespace.FIELD_NAME, "1"))); + assertEquals("can't resolve Symbol(namespace=FIELD_NAME, name=1) in type env", + exception.getMessage()); + } + + private TypeEnvironment environment() { + return context.peek(); + } + + private Symbol toSymbol(ReferenceExpression ref) { + return new Symbol(Namespace.FIELD_NAME, ref.getAttr()); + } + +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/symbol/SymbolTableTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/symbol/SymbolTableTest.java new file mode 100644 index 0000000000..93cc481667 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/symbol/SymbolTableTest.java @@ -0,0 +1,87 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.analysis.symbol; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BOOLEAN; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.hasEntry; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import java.util.Map; +import java.util.Optional; +import org.junit.jupiter.api.Test; + + +public class SymbolTableTest { + + private final SymbolTable symbolTable = new SymbolTable(); + + @Test + public void defineFieldSymbolShouldBeAbleToResolve() { + defineSymbolShouldBeAbleToResolve(new Symbol(Namespace.FIELD_NAME, "age"), INTEGER); + } + + + @Test + public void defineFieldSymbolShouldBeAbleToResolveByPrefix() { + symbolTable.store(new Symbol(Namespace.FIELD_NAME, "s.projects.active"), BOOLEAN); + symbolTable.store(new Symbol(Namespace.FIELD_NAME, "s.address"), STRING); + symbolTable.store(new Symbol(Namespace.FIELD_NAME, "s.manager.name"), STRING); + + Map typeByName = + symbolTable.lookupByPrefix(new Symbol(Namespace.FIELD_NAME, "s.projects")); + + assertThat( + typeByName, + allOf( + aMapWithSize(1), + hasEntry("s.projects.active", BOOLEAN) + ) + ); + } + + @Test + public void failedToResolveSymbolNoNamespaceMatched() { + symbolTable.store(new Symbol(Namespace.FUNCTION_NAME, "customFunction"), BOOLEAN); + assertFalse(symbolTable.lookup(new Symbol(Namespace.FIELD_NAME, "s.projects")).isPresent()); + + assertThat(symbolTable.lookupByPrefix(new Symbol(Namespace.FIELD_NAME, "s.projects")), + anEmptyMap()); + } + + @Test + public void isEmpty() { + symbolTable.store(new Symbol(Namespace.FUNCTION_NAME, "customFunction"), BOOLEAN); + assertTrue(symbolTable.isEmpty(Namespace.FIELD_NAME)); + } + + private void defineSymbolShouldBeAbleToResolve(Symbol symbol, ExprType expectedType) { + symbolTable.store(symbol, expectedType); + + Optional actualType = symbolTable.lookup(symbol); + assertTrue(actualType.isPresent()); + assertEquals(expectedType, actualType.get()); + } + +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/config/TestConfig.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/config/TestConfig.java new file mode 100644 index 0000000000..b38bc78377 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/config/TestConfig.java @@ -0,0 +1,108 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.config; + +import com.amazon.opendistroforelasticsearch.sql.analysis.symbol.Namespace; +import com.amazon.opendistroforelasticsearch.sql.analysis.symbol.Symbol; +import com.amazon.opendistroforelasticsearch.sql.analysis.symbol.SymbolTable; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; +import com.amazon.opendistroforelasticsearch.sql.storage.Table; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Configuration will be used for UT. + */ +@Configuration +public class TestConfig { + public static final String INT_TYPE_NULL_VALUE_FIELD = "int_null_value"; + public static final String INT_TYPE_MISSING_VALUE_FIELD = "int_missing_value"; + public static final String BOOL_TYPE_NULL_VALUE_FIELD = "null_value_boolean"; + public static final String BOOL_TYPE_MISSING_VALUE_FIELD = "missing_value_boolean"; + public static final String STRING_TYPE_NULL_VALUE_FILED = "string_null_value"; + public static final String STRING_TYPE_MISSING_VALUE_FILED = "string_missing_value"; + + private static Map typeMapping = new ImmutableMap.Builder() + .put("integer_value", ExprCoreType.INTEGER) + .put(INT_TYPE_NULL_VALUE_FIELD, ExprCoreType.INTEGER) + .put(INT_TYPE_MISSING_VALUE_FIELD, ExprCoreType.INTEGER) + .put("long_value", ExprCoreType.LONG) + .put("float_value", ExprCoreType.FLOAT) + .put("double_value", ExprCoreType.DOUBLE) + .put("boolean_value", ExprCoreType.BOOLEAN) + .put(BOOL_TYPE_NULL_VALUE_FIELD, ExprCoreType.BOOLEAN) + .put(BOOL_TYPE_MISSING_VALUE_FIELD, ExprCoreType.BOOLEAN) + .put("string_value", ExprCoreType.STRING) + .put(STRING_TYPE_NULL_VALUE_FILED, ExprCoreType.STRING) + .put(STRING_TYPE_MISSING_VALUE_FILED, ExprCoreType.STRING) + .put("struct_value", ExprCoreType.STRUCT) + .put("array_value", ExprCoreType.ARRAY) + .build(); + + @Bean + protected StorageEngine storageEngine() { + return new StorageEngine() { + @Override + public Table getTable(String name) { + return new Table() { + @Override + public Map getFieldTypes() { + return typeMapping; + } + + @Override + public PhysicalPlan implement(LogicalPlan plan) { + throw new UnsupportedOperationException(); + } + }; + } + }; + } + + + @Bean + protected SymbolTable symbolTable() { + SymbolTable symbolTable = new SymbolTable(); + typeMapping.entrySet() + .forEach( + entry -> symbolTable + .store(new Symbol(Namespace.FIELD_NAME, entry.getKey()), entry.getValue())); + return symbolTable; + } + + @Bean + protected Environment typeEnv() { + return var -> { + if (var instanceof ReferenceExpression) { + ReferenceExpression refExpr = (ReferenceExpression) var; + if (typeMapping.containsKey(refExpr.getAttr())) { + return typeMapping.get(refExpr.getAttr()); + } + } + throw new ExpressionEvaluationException("type resolved failed"); + }; + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprBooleanValueTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprBooleanValueTest.java new file mode 100644 index 0000000000..8e8e43dff1 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprBooleanValueTest.java @@ -0,0 +1,34 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import static com.amazon.opendistroforelasticsearch.sql.utils.ComparisonUtil.compare; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import org.junit.jupiter.api.Test; + +public class ExprBooleanValueTest { + + @Test + public void comparabilityTest() { + ExprValue booleanValue = ExprValueUtils.booleanValue(false); + ExpressionEvaluationException exception = assertThrows(ExpressionEvaluationException.class, + () -> compare(booleanValue, booleanValue)); + assertEquals("ExprBooleanValue instances are not comparable", exception.getMessage()); + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprCollectionValueTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprCollectionValueTest.java new file mode 100644 index 0000000000..64e932a0fe --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprCollectionValueTest.java @@ -0,0 +1,35 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import static com.amazon.opendistroforelasticsearch.sql.utils.ComparisonUtil.compare; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import java.util.Arrays; +import org.junit.jupiter.api.Test; + +public class ExprCollectionValueTest { + + @Test + public void comparabilityTest() { + ExprValue collectionValue = ExprValueUtils.collectionValue(Arrays.asList(0, 1)); + ExpressionEvaluationException exception = assertThrows(ExpressionEvaluationException.class, + () -> compare(collectionValue, collectionValue)); + assertEquals("ExprCollectionValue instances are not comparable", exception.getMessage()); + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprMissingValueTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprMissingValueTest.java new file mode 100644 index 0000000000..13cdd5f24f --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprMissingValueTest.java @@ -0,0 +1,55 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; +import static com.amazon.opendistroforelasticsearch.sql.utils.ComparisonUtil.compare; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import org.junit.jupiter.api.Test; + + +class ExprMissingValueTest { + + @Test + public void test_is_missing() { + assertTrue(LITERAL_MISSING.isMissing()); + } + + @Test + public void getValue() { + ExpressionEvaluationException exception = assertThrows(ExpressionEvaluationException.class, + () -> LITERAL_MISSING.value()); + assertEquals("invalid to call value operation on missing value", exception.getMessage()); + } + + @Test + public void getType() { + ExpressionEvaluationException exception = assertThrows(ExpressionEvaluationException.class, + () -> LITERAL_MISSING.type()); + assertEquals("invalid to call type operation on missing value", exception.getMessage()); + } + + @Test + public void comparabilityTest() { + ExpressionEvaluationException exception = assertThrows(ExpressionEvaluationException.class, + () -> compare(LITERAL_MISSING, LITERAL_MISSING)); + assertEquals("invalid to call compare operation on missing value", exception.getMessage()); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprNullValueTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprNullValueTest.java new file mode 100644 index 0000000000..8b9127fe56 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprNullValueTest.java @@ -0,0 +1,54 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static com.amazon.opendistroforelasticsearch.sql.utils.ComparisonUtil.compare; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import org.junit.jupiter.api.Test; + +public class ExprNullValueTest { + + @Test + public void test_is_null() { + assertTrue(LITERAL_NULL.isNull()); + } + + @Test + public void getValue() { + assertNull(LITERAL_NULL.value()); + } + + @Test + public void getType() { + ExpressionEvaluationException exception = assertThrows(ExpressionEvaluationException.class, + () -> LITERAL_NULL.type()); + assertEquals("invalid to call type operation on null value", exception.getMessage()); + } + + @Test + public void comparabilityTest() { + ExpressionEvaluationException exception = assertThrows(ExpressionEvaluationException.class, + () -> compare(LITERAL_NULL, LITERAL_NULL)); + assertEquals("invalid to call compare operation on null value", exception.getMessage()); + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTupleValueTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTupleValueTest.java new file mode 100644 index 0000000000..fcfb072b90 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTupleValueTest.java @@ -0,0 +1,58 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import static com.amazon.opendistroforelasticsearch.sql.utils.ComparisonUtil.compare; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Test; + +class ExprTupleValueTest { + @Test + public void equal_to_itself() { + ExprValue tupleValue = ExprValueUtils.tupleValue(ImmutableMap.of("integer_value", 2)); + assertTrue(tupleValue.equals(tupleValue)); + } + + @Test + public void tuple_compare_int() { + ExprValue tupleValue = ExprValueUtils.tupleValue(ImmutableMap.of("integer_value", 2)); + ExprValue intValue = ExprValueUtils.integerValue(10); + assertFalse(tupleValue.equals(intValue)); + } + + @Test + public void compre_tuple_with_different_size() { + ExprValue tupleValue1 = ExprValueUtils.tupleValue(ImmutableMap.of("integer_value", 2)); + ExprValue tupleValue2 = + ExprValueUtils.tupleValue(ImmutableMap.of("integer_value", 2, "float_value", 1f)); + assertFalse(tupleValue1.equals(tupleValue2)); + assertFalse(tupleValue2.equals(tupleValue1)); + } + + @Test + public void comparabilityTest() { + ExprValue tupleValue = ExprValueUtils.tupleValue(ImmutableMap.of("integer_value", 2)); + ExpressionEvaluationException exception = assertThrows(ExpressionEvaluationException.class, + () -> compare(tupleValue, tupleValue)); + assertEquals("ExprTupleValue instances are not comparable", exception.getMessage()); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtilsTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtilsTest.java new file mode 100644 index 0000000000..d0814ff672 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtilsTest.java @@ -0,0 +1,190 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.integerValue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.BindingTuple; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +@DisplayName("Test Expression Value Utils") +public class ExprValueUtilsTest { + private static List numberValues = Stream.of(1, 1L, 1f, 1D) + .map(ExprValueUtils::fromObjectValue).collect(Collectors.toList()); + private static List nonNumberValues = Stream.of("1", true, Arrays.asList(1), + ImmutableMap.of("1", 1)).map(ExprValueUtils::fromObjectValue).collect(Collectors.toList()); + private static List allValues = + Lists.newArrayList(Iterables.concat(numberValues, nonNumberValues)); + + private static List> numberValueExtractor = Arrays.asList( + ExprValueUtils::getIntegerValue, + ExprValueUtils::getLongValue, + ExprValueUtils::getFloatValue, + ExprValueUtils::getDoubleValue); + private static List> nonNumberValueExtractor = Arrays.asList( + ExprValueUtils::getStringValue, + ExprValueUtils::getBooleanValue, + ExprValueUtils::getCollectionValue, + ExprValueUtils::getTupleValue); + private static List> allValueExtractor = Lists.newArrayList( + Iterables.concat(numberValueExtractor, nonNumberValueExtractor)); + + private static List numberTypes = + Arrays.asList(ExprCoreType.INTEGER, ExprCoreType.LONG, ExprCoreType.FLOAT, + ExprCoreType.DOUBLE); + private static List nonNumberTypes = + Arrays.asList(ExprCoreType.STRING, ExprCoreType.BOOLEAN, ExprCoreType.ARRAY, + ExprCoreType.STRUCT); + private static List allTypes = + Lists.newArrayList(Iterables.concat(numberTypes, nonNumberTypes)); + + private static Stream getValueTestArgumentStream() { + List expectedValues = Arrays.asList(1, 1L, 1f, 1D, "1", true, + Arrays.asList(integerValue(1)), + ImmutableMap.of("1", integerValue(1))); + Stream.Builder builder = Stream.builder(); + for (int i = 0; i < expectedValues.size(); i++) { + builder.add(Arguments.of( + allValues.get(i), + allValueExtractor.get(i), + expectedValues.get(i))); + } + return builder.build(); + } + + private static Stream getTypeTestArgumentStream() { + Stream.Builder builder = Stream.builder(); + for (int i = 0; i < allValues.size(); i++) { + builder.add(Arguments.of( + allValues.get(i), + allTypes.get(i))); + } + return builder.build(); + } + + private static Stream invalidGetNumberValueArgumentStream() { + return Lists.cartesianProduct(nonNumberValues, numberValueExtractor) + .stream() + .map(list -> Arguments.of(list.get(0), list.get(1))); + } + + @SuppressWarnings("unchecked") + private static Stream invalidConvert() { + List, ExprCoreType>> extractorWithTypeList = + new ArrayList<>(); + for (int i = 0; i < nonNumberValueExtractor.size(); i++) { + extractorWithTypeList.add( + new AbstractMap.SimpleEntry<>(nonNumberValueExtractor.get(i), nonNumberTypes.get(i))); + } + return Lists.cartesianProduct(allValues, extractorWithTypeList) + .stream() + .filter(list -> { + ExprValue value = (ExprValue) list.get(0); + Map.Entry, ExprCoreType> entry = + (Map.Entry, + ExprCoreType>) list + .get(1); + return entry.getValue() != value.type(); + }) + .map(list -> { + Map.Entry, ExprCoreType> entry = + (Map.Entry, + ExprCoreType>) list + .get(1); + return Arguments.of(list.get(0), entry.getKey(), entry.getValue()); + }); + } + + @ParameterizedTest(name = "the value of ExprValue:{0} is: {2} ") + @MethodSource("getValueTestArgumentStream") + public void getValue(ExprValue value, Function extractor, Object expect) { + assertEquals(expect, extractor.apply(value)); + } + + @ParameterizedTest(name = "the type of ExprValue:{0} is: {1} ") + @MethodSource("getTypeTestArgumentStream") + public void getType(ExprValue value, ExprCoreType expectType) { + assertEquals(expectType, value.type()); + } + + /** + * Test Invalid to get number. + */ + @ParameterizedTest(name = "invalid to get number value of ExprValue:{0}") + @MethodSource("invalidGetNumberValueArgumentStream") + public void invalidGetNumberValue(ExprValue value, Function extractor) { + Exception exception = assertThrows(ExpressionEvaluationException.class, + () -> extractor.apply(value)); + assertEquals( + String.format("invalid to getNumberValue with expression has type of %s", value.type()), + exception.getMessage()); + } + + /** + * Test Invalid to convert. + */ + @ParameterizedTest(name = "invalid convert ExprValue:{0} to ExprType:{2}") + @MethodSource("invalidConvert") + public void invalidConvertExprValue(ExprValue value, Function extractor, + ExprCoreType toType) { + Exception exception = assertThrows(ExpressionEvaluationException.class, + () -> extractor.apply(value)); + assertEquals(String + .format("invalid to convert expression with type:%s to type:%s", value.type(), toType), + exception.getMessage()); + } + + @Test + public void unSupportedObject() { + Exception exception = assertThrows(ExpressionEvaluationException.class, + () -> ExprValueUtils.fromObjectValue(integerValue(1))); + assertEquals( + "unsupported object " + + "class com.amazon.opendistroforelasticsearch.sql.data.model.ExprIntegerValue", + exception.getMessage()); + } + + @Test + public void bindingTuples() { + for (ExprValue value : allValues) { + if (ExprCoreType.STRUCT == value.type()) { + assertNotEquals(BindingTuple.EMPTY, value.bindingTuples()); + } else { + assertEquals(BindingTuple.EMPTY, value.bindingTuples()); + } + } + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprTypeTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprTypeTest.java new file mode 100644 index 0000000000..7d75e79748 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprTypeTest.java @@ -0,0 +1,55 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.data.type; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.FLOAT; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.LONG; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.SHORT; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.UNKNOWN; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Test; + +class ExprTypeTest { + @Test + public void isCompatible() { + assertTrue(DOUBLE.isCompatible(DOUBLE)); + assertTrue(DOUBLE.isCompatible(FLOAT)); + assertTrue(DOUBLE.isCompatible(LONG)); + assertTrue(DOUBLE.isCompatible(INTEGER)); + assertTrue(DOUBLE.isCompatible(SHORT)); + assertTrue(FLOAT.isCompatible(FLOAT)); + assertTrue(FLOAT.isCompatible(LONG)); + assertTrue(FLOAT.isCompatible(INTEGER)); + assertTrue(FLOAT.isCompatible(SHORT)); + assertFalse(INTEGER.isCompatible(DOUBLE)); + assertFalse(STRING.isCompatible(DOUBLE)); + assertFalse(INTEGER.isCompatible(UNKNOWN)); + } + + @Test + public void getParent() { + assertThat(((ExprType) () -> "test").getParent(), Matchers.contains(UNKNOWN)); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprValueOrderingTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprValueOrderingTest.java new file mode 100644 index 0000000000..d2ac42afc4 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprValueOrderingTest.java @@ -0,0 +1,247 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.utils; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.collectionValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.doubleValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.floatValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.integerValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.longValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.stringValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.tupleValue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class ExprValueOrderingTest { + @Mock + ExprValue left; + @Mock + ExprValue right; + + @Test + public void natural() { + ExprValueOrdering ordering = ExprValueOrdering.natural(); + assertEquals(1, ordering.compare(integerValue(5), integerValue(4))); + assertEquals(0, ordering.compare(integerValue(5), integerValue(5))); + assertEquals(-1, ordering.compare(integerValue(4), integerValue(5))); + } + + @Test + public void natural_reverse() { + ExprValueOrdering ordering = ExprValueOrdering.natural().reverse(); + assertEquals(-1, ordering.compare(integerValue(5), integerValue(4))); + assertEquals(0, ordering.compare(integerValue(5), integerValue(5))); + assertEquals(1, ordering.compare(integerValue(4), integerValue(5))); + } + + @Test + public void natural_null_first() { + ExprValueOrdering ordering = ExprValueOrdering.natural().nullsFirst(); + assertEquals(1, ordering.compare(integerValue(5), LITERAL_NULL)); + assertEquals(1, ordering.compare(integerValue(5), LITERAL_MISSING)); + } + + @Test + public void natural_null_last() { + ExprValueOrdering ordering = ExprValueOrdering.natural().nullsLast(); + assertEquals(-1, ordering.compare(integerValue(5), LITERAL_NULL)); + assertEquals(-1, ordering.compare(integerValue(5), LITERAL_MISSING)); + assertEquals(1, ordering.compare(LITERAL_NULL, integerValue(5))); + assertEquals(1, ordering.compare(LITERAL_MISSING, integerValue(5))); + } + + @Test + public void natural_null_first_reverse() { + ExprValueOrdering ordering = ExprValueOrdering.natural().reverse().nullsFirst(); + assertEquals(-1, ordering.compare(integerValue(5), integerValue(4))); + assertEquals(0, ordering.compare(integerValue(5), integerValue(5))); + assertEquals(1, ordering.compare(integerValue(4), integerValue(5))); + assertEquals(1, ordering.compare(integerValue(5), LITERAL_NULL)); + assertEquals(1, ordering.compare(integerValue(5), LITERAL_MISSING)); + } + + @Test + public void natural_reverse_null_first() { + ExprValueOrdering ordering = ExprValueOrdering.natural().nullsFirst().reverse(); + assertEquals(-1, ordering.compare(integerValue(5), integerValue(4))); + assertEquals(0, ordering.compare(integerValue(5), integerValue(5))); + assertEquals(1, ordering.compare(integerValue(4), integerValue(5))); + assertEquals(-1, ordering.compare(integerValue(5), LITERAL_NULL)); + assertEquals(-1, ordering.compare(integerValue(5), LITERAL_MISSING)); + } + + @Test + public void natural_null_last_reverse() { + ExprValueOrdering ordering = ExprValueOrdering.natural().reverse().nullsLast(); + assertEquals(-1, ordering.compare(integerValue(5), integerValue(4))); + assertEquals(0, ordering.compare(integerValue(5), integerValue(5))); + assertEquals(1, ordering.compare(integerValue(4), integerValue(5))); + assertEquals(-1, ordering.compare(integerValue(5), LITERAL_NULL)); + assertEquals(-1, ordering.compare(integerValue(5), LITERAL_MISSING)); + } + + @Test + public void natural_reverse_null_last() { + ExprValueOrdering ordering = ExprValueOrdering.natural().nullsLast().reverse(); + assertEquals(-1, ordering.compare(integerValue(5), integerValue(4))); + assertEquals(0, ordering.compare(integerValue(5), integerValue(5))); + assertEquals(1, ordering.compare(integerValue(4), integerValue(5))); + assertEquals(1, ordering.compare(integerValue(5), LITERAL_NULL)); + assertEquals(1, ordering.compare(integerValue(5), LITERAL_MISSING)); + } + + @Test + public void natural_order_double_value() { + ExprValueOrdering ordering = ExprValueOrdering.natural(); + assertEquals(1, ordering.compare(doubleValue(5d), doubleValue(4d))); + assertEquals(0, ordering.compare(doubleValue(5d), doubleValue(5d))); + assertEquals(-1, ordering.compare(doubleValue(4d), doubleValue(5d))); + } + + @Test + public void natural_order_float_value() { + ExprValueOrdering ordering = ExprValueOrdering.natural(); + assertEquals(1, ordering.compare(floatValue(5f), floatValue(4f))); + assertEquals(0, ordering.compare(floatValue(5f), floatValue(5f))); + assertEquals(-1, ordering.compare(floatValue(4f), floatValue(5f))); + } + + @Test + public void natural_order_long_value() { + ExprValueOrdering ordering = ExprValueOrdering.natural(); + assertEquals(1, ordering.compare(longValue(5L), longValue(4L))); + assertEquals(0, ordering.compare(longValue(5L), longValue(5L))); + assertEquals(-1, ordering.compare(longValue(4L), longValue(5L))); + } + + @Test + public void natural_order_boolean_value() { + ExprValueOrdering ordering = ExprValueOrdering.natural(); + assertEquals(1, ordering.compare(LITERAL_TRUE, LITERAL_FALSE)); + assertEquals(0, ordering.compare(LITERAL_TRUE, LITERAL_TRUE)); + assertEquals(-1, ordering.compare(LITERAL_FALSE, LITERAL_TRUE)); + } + + @Test + public void natural_order_string_value() { + ExprValueOrdering ordering = ExprValueOrdering.natural(); + assertEquals(1, ordering.compare(stringValue("abd"), stringValue("abc"))); + assertEquals(0, ordering.compare(stringValue("abc"), stringValue("abc"))); + assertEquals(-1, ordering.compare(stringValue("abc"), stringValue("abd"))); + } + + @Test + public void natural_order_collection_value() { + ExprValueOrdering ordering = ExprValueOrdering.natural(); + assertEquals( + 1, + ordering.compare( + collectionValue(ImmutableList.of(1, 2)), collectionValue(ImmutableList.of(3)))); + assertEquals( + 0, + ordering.compare( + collectionValue(ImmutableList.of(1, 2)), collectionValue(ImmutableList.of(3, 4)))); + assertEquals( + -1, + ordering.compare( + collectionValue(ImmutableList.of(3)), collectionValue(ImmutableList.of(1, 2)))); + } + + @Test + public void natural_order_tuple_value() { + ExprValueOrdering ordering = ExprValueOrdering.natural(); + assertEquals( + 1, + ordering.compare( + tupleValue(ImmutableMap.of("v1", 1, "v2", 2)), tupleValue(ImmutableMap.of("v1", 3)))); + assertEquals( + 0, + ordering.compare( + tupleValue(ImmutableMap.of("v1", 1, "v2", 2)), + tupleValue(ImmutableMap.of("v1", 3, "v2", 4)))); + assertEquals( + -1, + ordering.compare( + tupleValue(ImmutableMap.of("v1", 3)), tupleValue(ImmutableMap.of("v1", 1, "v2", 2)))); + } + + @Test + public void order_compare_value_with_different_type() { + ExprValueOrdering ordering = ExprValueOrdering.natural(); + ExpressionEvaluationException exception = + assertThrows( + ExpressionEvaluationException.class, + () -> ordering.compare(integerValue(1), doubleValue(2d))); + assertEquals( + "compare expected value have same type, but with [INTEGER, DOUBLE]", + exception.getMessage()); + } + + @Test + public void order_compare_value_with_null_value() { + ExprValueOrdering ordering = ExprValueOrdering.natural(); + ExpressionEvaluationException exception = + assertThrows( + ExpressionEvaluationException.class, + () -> ordering.compare(integerValue(1), LITERAL_NULL)); + assertEquals("compare with null or missing value is invalid", exception.getMessage()); + + exception = + assertThrows( + ExpressionEvaluationException.class, + () -> ordering.compare(integerValue(1), LITERAL_MISSING)); + assertEquals("compare with null or missing value is invalid", exception.getMessage()); + + exception = + assertThrows( + ExpressionEvaluationException.class, + () -> ordering.compare(LITERAL_NULL, integerValue(1))); + assertEquals("compare with null or missing value is invalid", exception.getMessage()); + + exception = + assertThrows( + ExpressionEvaluationException.class, + () -> ordering.compare(LITERAL_MISSING, integerValue(1))); + assertEquals("compare with null or missing value is invalid", exception.getMessage()); + } + + @Test + public void order_compare_unknown_type() { + when(left.type()).thenReturn(ExprCoreType.UNKNOWN); + when(right.type()).thenReturn(ExprCoreType.UNKNOWN); + + ExprValueOrdering ordering = ExprValueOrdering.natural(); + ExpressionEvaluationException exception = + assertThrows(ExpressionEvaluationException.class, () -> ordering.compare(left, right)); + assertEquals("compare doesn't support type [UNKNOWN]", exception.getMessage()); + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/utils/NullsFirstExprValueOrderingTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/utils/NullsFirstExprValueOrderingTest.java new file mode 100644 index 0000000000..84f4c4de99 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/utils/NullsFirstExprValueOrderingTest.java @@ -0,0 +1,47 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.utils; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.integerValue; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import org.junit.jupiter.api.Test; + +class NullsFirstExprValueOrderingTest { + @Test + public void natural_null_first_null_first() { + ExprValueOrdering ordering = ExprValueOrdering.natural().nullsFirst().nullsFirst(); + assertEquals(1, ordering.compare(integerValue(5), LITERAL_NULL)); + assertEquals(1, ordering.compare(integerValue(5), LITERAL_MISSING)); + } + + @Test + public void natural_null_first_null_Last() { + ExprValueOrdering ordering = ExprValueOrdering.natural().nullsFirst().nullsLast(); + assertEquals(-1, ordering.compare(integerValue(5), LITERAL_NULL)); + assertEquals(-1, ordering.compare(integerValue(5), LITERAL_MISSING)); + } + + @Test + public void natural_null_first_compare_same_object() { + ExprValueOrdering ordering = ExprValueOrdering.natural().nullsFirst(); + ExprValue exprValue = integerValue(5); + assertEquals(0, ordering.compare(exprValue, exprValue)); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/utils/NullsLastExprValueOrderingTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/utils/NullsLastExprValueOrderingTest.java new file mode 100644 index 0000000000..47251506ad --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/utils/NullsLastExprValueOrderingTest.java @@ -0,0 +1,47 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.utils; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.integerValue; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import org.junit.jupiter.api.Test; + +class NullsLastExprValueOrderingTest { + @Test + public void natural_null_last_null_last() { + ExprValueOrdering ordering = ExprValueOrdering.natural().nullsLast().nullsLast(); + assertEquals(-1, ordering.compare(integerValue(5), LITERAL_NULL)); + assertEquals(-1, ordering.compare(integerValue(5), LITERAL_MISSING)); + } + + @Test + public void natural_null_last_null_first() { + ExprValueOrdering ordering = ExprValueOrdering.natural().nullsLast().nullsFirst(); + assertEquals(1, ordering.compare(integerValue(5), LITERAL_NULL)); + assertEquals(1, ordering.compare(integerValue(5), LITERAL_MISSING)); + } + + @Test + public void natural_null_last_compare_same_object() { + ExprValueOrdering ordering = ExprValueOrdering.natural().nullsLast(); + ExprValue exprValue = integerValue(5); + assertEquals(0, ordering.compare(exprValue, exprValue)); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ReverseExprValueOrderingTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ReverseExprValueOrderingTest.java new file mode 100644 index 0000000000..d8e07cc205 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ReverseExprValueOrderingTest.java @@ -0,0 +1,31 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.utils; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.integerValue; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class ReverseExprValueOrderingTest { + @Test + public void natural_reverse_reverse() { + ExprValueOrdering ordering = ExprValueOrdering.natural().reverse().reverse(); + assertEquals(1, ordering.compare(integerValue(5), integerValue(4))); + assertEquals(0, ordering.compare(integerValue(5), integerValue(5))); + assertEquals(-1, ordering.compare(integerValue(4), integerValue(5))); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/ExpressionTestBase.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/ExpressionTestBase.java new file mode 100644 index 0000000000..bfda027f26 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/ExpressionTestBase.java @@ -0,0 +1,123 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression; + +import static com.amazon.opendistroforelasticsearch.sql.config.TestConfig.BOOL_TYPE_MISSING_VALUE_FIELD; +import static com.amazon.opendistroforelasticsearch.sql.config.TestConfig.BOOL_TYPE_NULL_VALUE_FIELD; +import static com.amazon.opendistroforelasticsearch.sql.config.TestConfig.INT_TYPE_MISSING_VALUE_FIELD; +import static com.amazon.opendistroforelasticsearch.sql.config.TestConfig.INT_TYPE_NULL_VALUE_FIELD; +import static com.amazon.opendistroforelasticsearch.sql.config.TestConfig.STRING_TYPE_MISSING_VALUE_FILED; +import static com.amazon.opendistroforelasticsearch.sql.config.TestConfig.STRING_TYPE_NULL_VALUE_FILED; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.booleanValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.collectionValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.doubleValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.floatValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.integerValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.longValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.missingValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.nullValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.stringValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.tupleValue; + +import com.amazon.opendistroforelasticsearch.sql.config.TestConfig; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.expression.config.ExpressionConfig; +import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.util.List; +import java.util.function.Function; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +@Configuration +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {ExpressionConfig.class, ExpressionTestBase.class, + TestConfig.class}) +public class ExpressionTestBase { + @Autowired + protected DSL dsl; + + @Autowired + protected Environment typeEnv; + + @Bean + protected Environment valueEnv() { + return var -> { + if (var instanceof ReferenceExpression) { + switch (((ReferenceExpression) var).getAttr()) { + case "integer_value": + return integerValue(1); + case "long_value": + return longValue(1L); + case "float_value": + return floatValue(1f); + case "double_value": + return doubleValue(1d); + case "boolean_value": + return booleanValue(true); + case "string_value": + return stringValue("str"); + case "struct_value": + return tupleValue(ImmutableMap.of("str", 1)); + case "array_value": + return collectionValue(ImmutableList.of(1)); + case BOOL_TYPE_NULL_VALUE_FIELD: + case INT_TYPE_NULL_VALUE_FIELD: + case STRING_TYPE_NULL_VALUE_FILED: + return nullValue(); + case INT_TYPE_MISSING_VALUE_FIELD: + case BOOL_TYPE_MISSING_VALUE_FIELD: + case STRING_TYPE_MISSING_VALUE_FILED: + return missingValue(); + default: + throw new IllegalArgumentException("undefined reference"); + } + } else { + throw new IllegalArgumentException("var must be ReferenceExpression"); + } + }; + } + + @Bean + protected Environment typeEnv() { + return typeEnv; + } + + protected Function, FunctionExpression> functionMapping( + BuiltinFunctionName builtinFunctionName) { + switch (builtinFunctionName) { + case ADD: + return (expressions) -> dsl.add(expressions.get(0), expressions.get(1)); + case SUBTRACT: + return (expressions) -> dsl.subtract(expressions.get(0), expressions.get(1)); + case MULTIPLY: + return (expressions) -> dsl.multiply(expressions.get(0), expressions.get(1)); + case DIVIDE: + return (expressions) -> dsl.divide(expressions.get(0), expressions.get(1)); + case MODULES: + return (expressions) -> dsl.module(expressions.get(0), expressions.get(1)); + default: + throw new RuntimeException(); + } + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/ReferenceExpressionTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/ReferenceExpressionTest.java new file mode 100644 index 0000000000..a5d2b934f1 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/ReferenceExpressionTest.java @@ -0,0 +1,80 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression; + +import static com.amazon.opendistroforelasticsearch.sql.config.TestConfig.BOOL_TYPE_MISSING_VALUE_FIELD; +import static com.amazon.opendistroforelasticsearch.sql.config.TestConfig.BOOL_TYPE_NULL_VALUE_FIELD; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.booleanValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.collectionValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.doubleValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.floatValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.integerValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.longValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.stringValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.tupleValue; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.ARRAY; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BOOLEAN; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.FLOAT; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.LONG; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRUCT; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.DisplayNameGeneration; +import org.junit.jupiter.api.DisplayNameGenerator; +import org.junit.jupiter.api.Test; + +@DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) +class ReferenceExpressionTest extends ExpressionTestBase { + + @Test + public void resolve_value() { + assertEquals(integerValue(1), DSL.ref("integer_value", INTEGER).valueOf(valueEnv())); + assertEquals(longValue(1L), DSL.ref("long_value", LONG).valueOf(valueEnv())); + assertEquals(floatValue(1f), DSL.ref("float_value", FLOAT).valueOf(valueEnv())); + assertEquals(doubleValue(1d), DSL.ref("double_value", DOUBLE).valueOf(valueEnv())); + assertEquals(booleanValue(true), DSL.ref("boolean_value", BOOLEAN).valueOf(valueEnv())); + assertEquals(stringValue("str"), DSL.ref("string_value", STRING).valueOf(valueEnv())); + assertEquals(tupleValue(ImmutableMap.of("str", 1)), + DSL.ref("struct_value", STRUCT).valueOf(valueEnv())); + assertEquals(collectionValue(ImmutableList.of(1)), + DSL.ref("array_value", ARRAY).valueOf(valueEnv())); + assertEquals(LITERAL_NULL, DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN).valueOf(valueEnv())); + assertEquals(LITERAL_MISSING, + DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN).valueOf(valueEnv())); + } + + @Test + public void resolve_type() { + assertEquals(ExprCoreType.INTEGER, DSL.ref("integer_value", INTEGER).type()); + assertEquals(ExprCoreType.LONG, DSL.ref("long_value", LONG).type()); + assertEquals(ExprCoreType.FLOAT, DSL.ref("float_value", FLOAT).type()); + assertEquals(ExprCoreType.DOUBLE, DSL.ref("double_value", DOUBLE).type()); + assertEquals(ExprCoreType.BOOLEAN, DSL.ref("boolean_value", BOOLEAN).type()); + assertEquals(ExprCoreType.STRING, DSL.ref("string_value", STRING).type()); + assertEquals(ExprCoreType.STRUCT, DSL.ref("struct_value", STRUCT).type()); + assertEquals(ExprCoreType.ARRAY, DSL.ref("array_value", ARRAY).type()); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/AggregationTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/AggregationTest.java new file mode 100644 index 0000000000..ec42a97b2c --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/AggregationTest.java @@ -0,0 +1,94 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.aggregation; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.expression.ExpressionTestBase; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class AggregationTest extends ExpressionTestBase { + + protected static List tuples = + Arrays.asList( + ExprValueUtils.tupleValue( + new ImmutableMap.Builder() + .put("integer_value", 2) + .put("long_value", 2L) + .put("string_value", "m") + .put("double_value", 2d) + .put("float_value", 2f) + .put("boolean_value", true) + .put("struct_value", ImmutableMap.of("str", 1)) + .put("array_value", ImmutableList.of(1)) + .build()), + ExprValueUtils.tupleValue( + ImmutableMap.of( + "integer_value", + 1, + "long_value", + 1L, + "string_value", + "f", + "double_value", + 1d, + "float_value", + 1f)), + ExprValueUtils.tupleValue( + ImmutableMap.of( + "integer_value", + 3, + "long_value", + 3L, + "string_value", + "m", + "double_value", + 3d, + "float_value", + 3f)), + ExprValueUtils.tupleValue( + ImmutableMap.of( + "integer_value", + 4, + "long_value", + 4L, + "string_value", + "f", + "double_value", + 4d, + "float_value", + 4f))); + + protected static List tuples_with_null_and_missing = + Arrays.asList( + ExprValueUtils.tupleValue( + ImmutableMap.of("integer_value", 2, "string_value", "m", "double_value", 3d)), + ExprValueUtils.tupleValue( + ImmutableMap.of("integer_value", 1, "string_value", "f", "double_value", 4d)), + ExprValueUtils.tupleValue(Collections.singletonMap("double_value", null))); + + protected ExprValue aggregation(Aggregator aggregator, List tuples) { + AggregationState state = aggregator.create(); + for (ExprValue tuple : tuples) { + aggregator.iterate(tuple.bindingTuples(), state); + } + return state.result(); + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/AvgAggregatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/AvgAggregatorTest.java new file mode 100644 index 0000000000..cc54b5cef5 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/AvgAggregatorTest.java @@ -0,0 +1,80 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.aggregation; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import org.junit.jupiter.api.Test; + +class AvgAggregatorTest extends AggregationTest { + + @Test + public void avg_field_expression() { + ExprValue result = aggregation(dsl.avg(DSL.ref("integer_value", INTEGER)), tuples); + assertEquals(2.5, result.value()); + } + + @Test + public void avg_arithmetic_expression() { + ExprValue result = aggregation(dsl.avg( + dsl.multiply(DSL.ref("integer_value", INTEGER), + DSL.literal(ExprValueUtils.integerValue(10)))), tuples); + assertEquals(25.0, result.value()); + } + + @Test + public void avg_with_missing() { + ExprValue result = + aggregation(dsl.avg(DSL.ref("integer_value", INTEGER)), tuples_with_null_and_missing); + assertTrue(result.isNull()); + } + + @Test + public void avg_with_null() { + ExprValue result = + aggregation(dsl.avg(DSL.ref("double_value", DOUBLE)), tuples_with_null_and_missing); + assertTrue(result.isNull()); + } + + @Test + public void valueOf() { + ExpressionEvaluationException exception = assertThrows(ExpressionEvaluationException.class, + () -> dsl.avg(DSL.ref("double_value", DOUBLE)).valueOf(valueEnv())); + assertEquals("can't evaluate on aggregator: avg", exception.getMessage()); + } + + @Test + public void test_to_string() { + Aggregator avgAggregator = dsl.avg(DSL.ref("integer_value", INTEGER)); + assertEquals("avg(integer_value)", avgAggregator.toString()); + } + + @Test + public void test_nested_to_string() { + Aggregator avgAggregator = dsl.avg(dsl.multiply(DSL.ref("integer_value", INTEGER), + DSL.literal(ExprValueUtils.integerValue(10)))); + assertEquals(String.format("avg(%s * %d)", DSL.ref("integer_value", INTEGER), 10), + avgAggregator.toString()); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/CountAggregatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/CountAggregatorTest.java new file mode 100644 index 0000000000..5137170ea2 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/CountAggregatorTest.java @@ -0,0 +1,126 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.aggregation; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.ARRAY; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BOOLEAN; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.FLOAT; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.LONG; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRUCT; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import org.junit.jupiter.api.Test; + +class CountAggregatorTest extends AggregationTest { + + @Test + public void count_integer_field_expression() { + ExprValue result = aggregation(dsl.count(DSL.ref("integer_value", INTEGER)), tuples); + assertEquals(4, result.value()); + } + + @Test + public void count_long_field_expression() { + ExprValue result = aggregation(dsl.count(DSL.ref("long_value", LONG)), tuples); + assertEquals(4, result.value()); + } + + @Test + public void count_float_field_expression() { + ExprValue result = aggregation(dsl.count(DSL.ref("float_value", FLOAT)), tuples); + assertEquals(4, result.value()); + } + + @Test + public void count_double_field_expression() { + ExprValue result = aggregation(dsl.count(DSL.ref("double_value", DOUBLE)), tuples); + assertEquals(4, result.value()); + } + + @Test + public void count_arithmetic_expression() { + ExprValue result = aggregation(dsl.count( + dsl.multiply(DSL.ref("integer_value", INTEGER), + DSL.literal(ExprValueUtils.integerValue(10)))), tuples); + assertEquals(4, result.value()); + } + + @Test + public void count_string_field_expression() { + ExprValue result = aggregation(dsl.count(DSL.ref("string_value", STRING)), tuples); + assertEquals(4, result.value()); + } + + @Test + public void count_boolean_field_expression() { + ExprValue result = aggregation(dsl.count(DSL.ref("boolean_value", BOOLEAN)), tuples); + assertEquals(1, result.value()); + } + + @Test + public void count_struct_field_expression() { + ExprValue result = aggregation(dsl.count(DSL.ref("struct_value", STRUCT)), tuples); + assertEquals(1, result.value()); + } + + @Test + public void count_array_field_expression() { + ExprValue result = aggregation(dsl.count(DSL.ref("array_value", ARRAY)), tuples); + assertEquals(1, result.value()); + } + + @Test + public void count_with_missing() { + ExprValue result = aggregation(dsl.count(DSL.ref("integer_value", INTEGER)), + tuples_with_null_and_missing); + assertEquals(2, result.value()); + } + + @Test + public void count_with_null() { + ExprValue result = aggregation(dsl.count(DSL.ref("double_value", DOUBLE)), + tuples_with_null_and_missing); + assertEquals(2, result.value()); + } + + @Test + public void valueOf() { + ExpressionEvaluationException exception = assertThrows(ExpressionEvaluationException.class, + () -> dsl.count(DSL.ref("double_value", DOUBLE)).valueOf(valueEnv())); + assertEquals("can't evaluate on aggregator: count", exception.getMessage()); + } + + @Test + public void test_to_string() { + Aggregator countAggregator = dsl.count(DSL.ref("integer_value", INTEGER)); + assertEquals("count(integer_value)", countAggregator.toString()); + } + + @Test + public void test_nested_to_string() { + Aggregator countAggregator = dsl.count(dsl.abs(DSL.ref("integer_value", INTEGER))); + assertEquals(String.format("count(abs(%s))", DSL.ref("integer_value", INTEGER)), + countAggregator.toString()); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/SumAggregatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/SumAggregatorTest.java new file mode 100644 index 0000000000..c35e1c7832 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/SumAggregatorTest.java @@ -0,0 +1,119 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.aggregation; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.FLOAT; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.LONG; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.SumAggregator.SumState; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Test; + +class SumAggregatorTest extends AggregationTest { + + @Test + public void sum_integer_field_expression() { + ExprValue result = aggregation(dsl.sum(DSL.ref("integer_value", INTEGER)), tuples); + assertEquals(10, result.value()); + } + + @Test + public void sum_long_field_expression() { + ExprValue result = aggregation(dsl.sum(DSL.ref("long_value", LONG)), tuples); + assertEquals(10L, result.value()); + } + + @Test + public void sum_float_field_expression() { + ExprValue result = aggregation(dsl.sum(DSL.ref("float_value", FLOAT)), tuples); + assertEquals(10f, result.value()); + } + + @Test + public void sum_double_field_expression() { + ExprValue result = aggregation(dsl.sum(DSL.ref("double_value", DOUBLE)), tuples); + assertEquals(10d, result.value()); + } + + @Test + public void sum_arithmetic_expression() { + ExprValue result = aggregation(dsl.sum( + dsl.multiply(DSL.ref("integer_value", INTEGER), + DSL.literal(ExprValueUtils.integerValue(10)))), tuples); + assertEquals(100, result.value()); + } + + @Test + public void sum_string_field_expression() { + SumAggregator sumAggregator = + new SumAggregator(ImmutableList.of(DSL.ref("string_value", STRING)), ExprCoreType.STRING); + SumState sumState = sumAggregator.create(); + ExpressionEvaluationException exception = assertThrows(ExpressionEvaluationException.class, + () -> sumAggregator + .iterate( + ExprValueUtils.tupleValue(ImmutableMap.of("string_value", "m")).bindingTuples(), + sumState) + ); + assertEquals("unexpected type [STRING] in sum aggregation", exception.getMessage()); + } + + @Test + public void sum_with_missing() { + ExprValue result = + aggregation(dsl.sum(DSL.ref("integer_value", INTEGER)), tuples_with_null_and_missing); + assertTrue(result.isNull()); + } + + @Test + public void sum_with_null() { + ExprValue result = + aggregation(dsl.sum(DSL.ref("double_value", DOUBLE)), tuples_with_null_and_missing); + assertTrue(result.isNull()); + } + + @Test + public void valueOf() { + ExpressionEvaluationException exception = assertThrows(ExpressionEvaluationException.class, + () -> dsl.sum(DSL.ref("double_value", DOUBLE)).valueOf(valueEnv())); + assertEquals("can't evaluate on aggregator: sum", exception.getMessage()); + } + + @Test + public void test_to_string() { + Aggregator sumAggregator = dsl.sum(DSL.ref("integer_value", INTEGER)); + assertEquals("sum(integer_value)", sumAggregator.toString()); + } + + @Test + public void test_nested_to_string() { + Aggregator sumAggregator = dsl.sum(dsl.multiply(DSL.ref("integer_value", INTEGER), + DSL.literal(ExprValueUtils.integerValue(10)))); + assertEquals(String.format("sum(%s * %d)", DSL.ref("integer_value", INTEGER), 10), + sumAggregator.toString()); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionNameTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionNameTest.java new file mode 100644 index 0000000000..6feeb831b4 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionNameTest.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.function; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Arrays; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class BuiltinFunctionNameTest { + + private static Stream ofArguments() { + Stream.Builder builder = Stream.builder(); + return Arrays.asList(BuiltinFunctionName.values()) + .stream() + .map(functionName -> Arguments.of(functionName.getName().getFunctionName(), functionName)); + } + + @ParameterizedTest + @MethodSource("ofArguments") + public void of(String name, BuiltinFunctionName expected) { + assertTrue(BuiltinFunctionName.of(name).isPresent()); + assertEquals(expected, BuiltinFunctionName.of(name).get()); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionRepositoryTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionRepositoryTest.java new file mode 100644 index 0000000000..a60e83a3ad --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionRepositoryTest.java @@ -0,0 +1,101 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.function; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment; +import java.util.Arrays; +import java.util.Map; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class BuiltinFunctionRepositoryTest { + @Mock + private FunctionResolver mockfunctionResolver; + @Mock + private Map mockMap; + @Mock + private FunctionName mockFunctionName; + @Mock + private FunctionBuilder functionExpressionBuilder; + @Mock + private FunctionSignature functionSignature; + @Mock + private Expression mockExpression; + @Mock + private Environment emptyEnv; + + @Test + void register() { + BuiltinFunctionRepository repo = new BuiltinFunctionRepository(mockMap); + when(mockfunctionResolver.getFunctionName()).thenReturn(mockFunctionName); + repo.register(mockfunctionResolver); + + verify(mockMap, times(1)).put(mockFunctionName, mockfunctionResolver); + } + + @Test + void compile() { + when(mockfunctionResolver.getFunctionName()).thenReturn(mockFunctionName); + when(mockfunctionResolver.resolve(any())).thenReturn(functionExpressionBuilder); + when(mockMap.containsKey(any())).thenReturn(true); + when(mockMap.get(any())).thenReturn(mockfunctionResolver); + BuiltinFunctionRepository repo = new BuiltinFunctionRepository(mockMap); + repo.register(mockfunctionResolver); + + repo.compile(mockFunctionName, Arrays.asList(mockExpression)); + verify(functionExpressionBuilder, times(1)).apply(any()); + } + + @Test + @DisplayName("resolve registered function should pass") + void resolve() { + when(functionSignature.getFunctionName()).thenReturn(mockFunctionName); + when(mockfunctionResolver.getFunctionName()).thenReturn(mockFunctionName); + when(mockfunctionResolver.resolve(functionSignature)).thenReturn(functionExpressionBuilder); + when(mockMap.containsKey(mockFunctionName)).thenReturn(true); + when(mockMap.get(mockFunctionName)).thenReturn(mockfunctionResolver); + BuiltinFunctionRepository repo = new BuiltinFunctionRepository(mockMap); + repo.register(mockfunctionResolver); + + assertEquals(functionExpressionBuilder, repo.resolve(functionSignature)); + } + + @Test + @DisplayName("resolve unregistered function should throw exception") + void resolve_unregistered() { + BuiltinFunctionRepository repo = new BuiltinFunctionRepository(mockMap); + when(mockMap.containsKey(any())).thenReturn(false); + repo.register(mockfunctionResolver); + + ExpressionEvaluationException exception = assertThrows(ExpressionEvaluationException.class, + () -> repo.resolve(new FunctionSignature(FunctionName.of("unknown"), Arrays.asList()))); + assertEquals("unsupported function name: unknown", exception.getMessage()); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionResolverTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionResolverTest.java new file mode 100644 index 0000000000..d38ba060c7 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionResolverTest.java @@ -0,0 +1,88 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.function; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.data.type.WideningTypeRule; +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.DisplayNameGeneration; +import org.junit.jupiter.api.DisplayNameGenerator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) +@ExtendWith(MockitoExtension.class) +class FunctionResolverTest { + @Mock + private FunctionSignature exactlyMatchFS; + @Mock + private FunctionSignature bestMatchFS; + @Mock + private FunctionSignature leastMatchFS; + @Mock + private FunctionSignature notMatchFS; + @Mock + private FunctionSignature functionSignature; + @Mock + private FunctionBuilder exactlyMatchBuilder; + @Mock + private FunctionBuilder bestMatchBuilder; + @Mock + private FunctionBuilder leastMatchBuilder; + @Mock + private FunctionBuilder notMatchBuilder; + + private FunctionName functionName = FunctionName.of("add"); + + @Test + void resolve_function_signature_exactly_match() { + when(functionSignature.match(exactlyMatchFS)).thenReturn(WideningTypeRule.TYPE_EQUAL); + FunctionResolver resolver = new FunctionResolver(functionName, + ImmutableMap.of(exactlyMatchFS, exactlyMatchBuilder)); + + assertEquals(exactlyMatchBuilder, resolver.resolve(functionSignature)); + } + + @Test + void resolve_function_signature_best_match() { + when(functionSignature.match(bestMatchFS)).thenReturn(1); + when(functionSignature.match(leastMatchFS)).thenReturn(2); + FunctionResolver resolver = new FunctionResolver(functionName, + ImmutableMap.of(bestMatchFS, bestMatchBuilder, leastMatchFS, leastMatchBuilder)); + + assertEquals(bestMatchBuilder, resolver.resolve(functionSignature)); + } + + @Test + void resolve_function_not_match() { + when(functionSignature.match(notMatchFS)).thenReturn(WideningTypeRule.IMPOSSIBLE_WIDENING); + when(notMatchFS.formatTypes()).thenReturn("[INTEGER,INTEGER]"); + when(functionSignature.formatTypes()).thenReturn("[BOOLEAN,BOOLEAN]"); + FunctionResolver resolver = new FunctionResolver(functionName, + ImmutableMap.of(notMatchFS, notMatchBuilder)); + + ExpressionEvaluationException exception = assertThrows(ExpressionEvaluationException.class, + () -> resolver.resolve(functionSignature)); + assertEquals("add function expected {[INTEGER,INTEGER]}, but get [BOOLEAN,BOOLEAN]", + exception.getMessage()); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionSignatureTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionSignatureTest.java new file mode 100644 index 0000000000..7a5b8b16a4 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionSignatureTest.java @@ -0,0 +1,105 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.function; + +import static com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionSignature.EXACTLY_MATCH; +import static com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionSignature.NOT_MATCH; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.DisplayNameGeneration; +import org.junit.jupiter.api.DisplayNameGenerator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) +@ExtendWith(MockitoExtension.class) +class FunctionSignatureTest { + @Mock + private FunctionSignature funcSignature; + @Mock + private List funcParamTypeList; + + private FunctionName unresolvedFuncName = FunctionName.of("add"); + private List unresolvedParamTypeList = + Arrays.asList(ExprCoreType.INTEGER, ExprCoreType.FLOAT); + + @Test + void signature_name_not_match() { + when(funcSignature.getFunctionName()).thenReturn(FunctionName.of(("diff"))); + FunctionSignature unresolvedFunSig = + new FunctionSignature(this.unresolvedFuncName, unresolvedParamTypeList); + + assertEquals(NOT_MATCH, unresolvedFunSig.match(funcSignature)); + } + + @Test + void signature_arguments_size_not_match() { + when(funcSignature.getFunctionName()).thenReturn(unresolvedFuncName); + when(funcSignature.getParamTypeList()).thenReturn(funcParamTypeList); + when(funcParamTypeList.size()).thenReturn(1); + FunctionSignature unresolvedFunSig = + new FunctionSignature(unresolvedFuncName, unresolvedParamTypeList); + + assertEquals(NOT_MATCH, unresolvedFunSig.match(funcSignature)); + } + + @Test + void signature_exactly_match() { + when(funcSignature.getFunctionName()).thenReturn(unresolvedFuncName); + when(funcSignature.getParamTypeList()).thenReturn(unresolvedParamTypeList); + FunctionSignature unresolvedFunSig = + new FunctionSignature(unresolvedFuncName, unresolvedParamTypeList); + + assertEquals(EXACTLY_MATCH, unresolvedFunSig.match(funcSignature)); + } + + @Test + void signature_not_match() { + when(funcSignature.getFunctionName()).thenReturn(unresolvedFuncName); + when(funcSignature.getParamTypeList()) + .thenReturn(Arrays.asList(ExprCoreType.STRING, ExprCoreType.STRING)); + FunctionSignature unresolvedFunSig = + new FunctionSignature(unresolvedFuncName, unresolvedParamTypeList); + + assertEquals(NOT_MATCH, unresolvedFunSig.match(funcSignature)); + } + + @Test + void signature_widening_match() { + when(funcSignature.getFunctionName()).thenReturn(unresolvedFuncName); + when(funcSignature.getParamTypeList()) + .thenReturn(Arrays.asList(ExprCoreType.FLOAT, ExprCoreType.FLOAT)); + FunctionSignature unresolvedFunSig = + new FunctionSignature(unresolvedFuncName, unresolvedParamTypeList); + + assertEquals(2, unresolvedFunSig.match(funcSignature)); + } + + @Test + void format_types() { + FunctionSignature unresolvedFunSig = + new FunctionSignature(unresolvedFuncName, unresolvedParamTypeList); + + assertEquals("[INTEGER,FLOAT]", unresolvedFunSig.formatTypes()); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/function/WideningTypeRuleTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/function/WideningTypeRuleTest.java new file mode 100644 index 0000000000..8b4f023ab3 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/function/WideningTypeRuleTest.java @@ -0,0 +1,114 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.function; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.FLOAT; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.LONG; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.SHORT; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.UNKNOWN; +import static com.amazon.opendistroforelasticsearch.sql.data.type.WideningTypeRule.IMPOSSIBLE_WIDENING; +import static com.amazon.opendistroforelasticsearch.sql.data.type.WideningTypeRule.TYPE_EQUAL; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.data.type.WideningTypeRule; +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import com.google.common.collect.ImmutableTable; +import com.google.common.collect.Lists; +import com.google.common.collect.Table; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class WideningTypeRuleTest { + private static Table numberWidenRule = + new ImmutableTable.Builder() + .put(SHORT, INTEGER, 1) + .put(SHORT, LONG, 2) + .put(SHORT, FLOAT, 3) + .put(SHORT, DOUBLE, 4) + .put(INTEGER, LONG, 1) + .put(INTEGER, FLOAT, 2) + .put(INTEGER, DOUBLE, 3) + .put(LONG, FLOAT, 1) + .put(LONG, DOUBLE, 2) + .put(FLOAT, DOUBLE, 1) + .build(); + + private static Stream distanceArguments() { + List exprTypes = + Arrays.asList(ExprCoreType.values()).stream().filter(type -> type != UNKNOWN).collect( + Collectors.toList()); + return Lists.cartesianProduct(exprTypes, exprTypes).stream() + .map(list -> { + ExprCoreType type1 = list.get(0); + ExprCoreType type2 = list.get(1); + if (type1 == type2) { + return Arguments.of(type1, type2, TYPE_EQUAL); + } else if (numberWidenRule.contains(type1, type2)) { + return Arguments.of(type1, type2, numberWidenRule.get(type1, type2)); + } else { + return Arguments.of(type1, type2, IMPOSSIBLE_WIDENING); + } + }); + } + + private static Stream validMaxTypes() { + List exprTypes = + Arrays.asList(ExprCoreType.values()).stream().filter(type -> type != UNKNOWN).collect( + Collectors.toList()); + return Lists.cartesianProduct(exprTypes, exprTypes).stream() + .map(list -> { + ExprCoreType type1 = list.get(0); + ExprCoreType type2 = list.get(1); + if (type1 == type2) { + return Arguments.of(type1, type2, type1); + } else if (numberWidenRule.contains(type1, type2)) { + return Arguments.of(type1, type2, type2); + } else if (numberWidenRule.contains(type2, type1)) { + return Arguments.of(type1, type2, type1); + } else { + return Arguments.of(type1, type2, null); + } + }); + } + + @ParameterizedTest + @MethodSource("distanceArguments") + public void distance(ExprCoreType v1, ExprCoreType v2, Integer expected) { + assertEquals(expected, WideningTypeRule.distance(v1, v2)); + } + + @ParameterizedTest + @MethodSource("validMaxTypes") + public void max(ExprCoreType v1, ExprCoreType v2, ExprCoreType expected) { + if (null == expected) { + ExpressionEvaluationException exception = assertThrows( + ExpressionEvaluationException.class, () -> WideningTypeRule.max(v1, v2)); + assertEquals(String.format("no max type of %s and %s ", v1, v2), exception.getMessage()); + } else { + assertEquals(expected, WideningTypeRule.max(v1, v2)); + } + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/ArithmeticFunctionTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/ArithmeticFunctionTest.java new file mode 100644 index 0000000000..002d2f1b19 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/ArithmeticFunctionTest.java @@ -0,0 +1,285 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.operator.arthmetic; + +import static com.amazon.opendistroforelasticsearch.sql.config.TestConfig.INT_TYPE_MISSING_VALUE_FIELD; +import static com.amazon.opendistroforelasticsearch.sql.config.TestConfig.INT_TYPE_NULL_VALUE_FIELD; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.integerValue; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.literal; +import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.ref; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.data.type.WideningTypeRule; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.ExpressionTestBase; +import com.amazon.opendistroforelasticsearch.sql.expression.FunctionExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName; +import com.google.common.collect.Lists; +import java.util.Arrays; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.jupiter.api.DisplayNameGeneration; +import org.junit.jupiter.api.DisplayNameGenerator; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +@DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) +class ArithmeticFunctionTest extends ExpressionTestBase { + + private static Stream arithmeticFunctionArguments() { + List numberOp1 = Stream.of(3, 3L, 3f, 3D) + .map(ExprValueUtils::fromObjectValue).collect(Collectors.toList()); + List numberOp2 = Stream.of(2, 2L, 2f, 2D) + .map(ExprValueUtils::fromObjectValue).collect(Collectors.toList()); + return Lists.cartesianProduct(numberOp1, numberOp2).stream() + .map(list -> Arguments.of(list.get(0), list.get(1))); + } + + private static Stream arithmeticOperatorArguments() { + return Stream + .of(BuiltinFunctionName.ADD, BuiltinFunctionName.SUBTRACT, BuiltinFunctionName.MULTIPLY, + BuiltinFunctionName.DIVIDE, BuiltinFunctionName.DIVIDE).map(Arguments::of); + } + + @ParameterizedTest(name = "add({1}, {2})") + @MethodSource("arithmeticFunctionArguments") + public void add(ExprValue op1, ExprValue op2) { + FunctionExpression expression = dsl.add(literal(op1), literal(op2)); + ExprType expectedType = WideningTypeRule.max(op1.type(), op2.type()); + assertEquals(expectedType, expression.type()); + assertValueEqual(BuiltinFunctionName.ADD, expectedType, op1, op2, expression.valueOf(null)); + assertEquals(String.format("%s + %s", op1.toString(), op2.toString()), expression.toString()); + } + + @ParameterizedTest(name = "{0}(int,null)") + @MethodSource("arithmeticOperatorArguments") + public void arithmetic_int_null(BuiltinFunctionName builtinFunctionName) { + Function< + List, FunctionExpression> function = functionMapping(builtinFunctionName); + + FunctionExpression functionExpression = + function.apply(Arrays.asList(literal(integerValue(1)), + ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER))); + assertEquals(INTEGER, functionExpression.type()); + assertEquals(LITERAL_NULL, functionExpression.valueOf(valueEnv())); + + functionExpression = function.apply( + Arrays.asList(ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER), literal(integerValue(1)))); + assertEquals(INTEGER, functionExpression.type()); + assertEquals(LITERAL_NULL, functionExpression.valueOf(valueEnv())); + } + + @ParameterizedTest(name = "{0}(int,missing)") + @MethodSource("arithmeticOperatorArguments") + public void arithmetic_int_missing(BuiltinFunctionName builtinFunctionName) { + Function< + List, FunctionExpression> function = functionMapping(builtinFunctionName); + FunctionExpression functionExpression = + function.apply(Arrays.asList(literal(integerValue(1)), + ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER))); + assertEquals(INTEGER, functionExpression.type()); + assertEquals(LITERAL_MISSING, functionExpression.valueOf(valueEnv())); + + functionExpression = function.apply(Arrays.asList(ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER), + literal(integerValue(1)))); + assertEquals(INTEGER, functionExpression.type()); + assertEquals(LITERAL_MISSING, functionExpression.valueOf(valueEnv())); + } + + @ParameterizedTest(name = "{0}(null,missing)") + @MethodSource("arithmeticOperatorArguments") + public void arithmetic_null_missing(BuiltinFunctionName builtinFunctionName) { + Function< + List, FunctionExpression> function = functionMapping(builtinFunctionName); + FunctionExpression functionExpression = function.apply( + Arrays.asList(ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER), + ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER))); + assertEquals(INTEGER, functionExpression.type()); + assertEquals(LITERAL_NULL, functionExpression.valueOf(valueEnv())); + + functionExpression = function.apply( + Arrays.asList(ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER), + ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER))); + assertEquals(INTEGER, functionExpression.type()); + assertEquals(LITERAL_MISSING, functionExpression.valueOf(valueEnv())); + + functionExpression = function.apply( + Arrays.asList(ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER), + ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER))); + assertEquals(INTEGER, functionExpression.type()); + assertEquals(LITERAL_MISSING, functionExpression.valueOf(valueEnv())); + + functionExpression = function.apply( + Arrays.asList(ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER), + ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER))); + assertEquals(INTEGER, functionExpression.type()); + assertEquals(LITERAL_MISSING, functionExpression.valueOf(valueEnv())); + } + + @ParameterizedTest(name = "subtract({1}, {2})") + @MethodSource("arithmeticFunctionArguments") + public void subtract(ExprValue op1, ExprValue op2) { + FunctionExpression expression = dsl.subtract(literal(op1), literal(op2)); + ExprType expectedType = WideningTypeRule.max(op1.type(), op2.type()); + assertEquals(expectedType, expression.type()); + assertValueEqual(BuiltinFunctionName.SUBTRACT, expectedType, op1, op2, + expression.valueOf(null)); + assertEquals(String.format("%s - %s", op1.toString(), op2.toString()), expression.toString()); + } + + @ParameterizedTest(name = "multiply({1}, {2})") + @MethodSource("arithmeticFunctionArguments") + public void multiply(ExprValue op1, ExprValue op2) { + FunctionExpression expression = dsl.multiply(literal(op1), literal(op2)); + ExprType expectedType = WideningTypeRule.max(op1.type(), op2.type()); + assertEquals(expectedType, expression.type()); + assertValueEqual(BuiltinFunctionName.MULTIPLY, expectedType, op1, op2, + expression.valueOf(null)); + assertEquals(String.format("%s * %s", op1.toString(), op2.toString()), expression.toString()); + } + + @ParameterizedTest(name = "divide({1}, {2})") + @MethodSource("arithmeticFunctionArguments") + public void divide(ExprValue op1, ExprValue op2) { + FunctionExpression expression = dsl.divide(literal(op1), literal(op2)); + ExprType expectedType = WideningTypeRule.max(op1.type(), op2.type()); + assertEquals(expectedType, expression.type()); + assertValueEqual(BuiltinFunctionName.DIVIDE, expectedType, op1, op2, expression.valueOf(null)); + assertEquals(String.format("%s / %s", op1.toString(), op2.toString()), expression.toString()); + } + + @ParameterizedTest(name = "module({1}, {2})") + @MethodSource("arithmeticFunctionArguments") + public void module(ExprValue op1, ExprValue op2) { + FunctionExpression expression = dsl.module(literal(op1), literal(op2)); + ExprType expectedType = WideningTypeRule.max(op1.type(), op2.type()); + assertEquals(expectedType, expression.type()); + assertValueEqual(BuiltinFunctionName.MODULES, expectedType, op1, op2, expression.valueOf(null)); + assertEquals(op1.toString() + " % " + op2.toString(), expression.toString()); + } + + protected void assertValueEqual(BuiltinFunctionName builtinFunctionName, ExprType type, + ExprValue op1, + ExprValue op2, + ExprValue actual) { + switch ((ExprCoreType) type) { + case INTEGER: + Integer vi1 = ExprValueUtils.getIntegerValue(op1); + Integer vi2 = ExprValueUtils.getIntegerValue(op2); + Integer viActual = ExprValueUtils.getIntegerValue(actual); + switch (builtinFunctionName) { + case ADD: + assertEquals(vi1 + vi2, viActual); + return; + case SUBTRACT: + assertEquals(vi1 - vi2, viActual); + return; + case DIVIDE: + assertEquals(vi1 / vi2, viActual); + return; + case MULTIPLY: + assertEquals(vi1 * vi2, viActual); + return; + case MODULES: + assertEquals(vi1 % vi2, viActual); + return; + default: + throw new IllegalStateException("illegal function name " + builtinFunctionName); + } + case LONG: + Long vl1 = ExprValueUtils.getLongValue(op1); + Long vl2 = ExprValueUtils.getLongValue(op2); + Long vlActual = ExprValueUtils.getLongValue(actual); + switch (builtinFunctionName) { + case ADD: + assertEquals(vl1 + vl2, vlActual); + return; + case SUBTRACT: + assertEquals(vl1 - vl2, vlActual); + return; + case DIVIDE: + assertEquals(vl1 / vl2, vlActual); + return; + case MULTIPLY: + assertEquals(vl1 * vl2, vlActual); + return; + case MODULES: + assertEquals(vl1 % vl2, vlActual); + return; + default: + throw new IllegalStateException("illegal function name " + builtinFunctionName); + } + case FLOAT: + Float vf1 = ExprValueUtils.getFloatValue(op1); + Float vf2 = ExprValueUtils.getFloatValue(op2); + Float vfActual = ExprValueUtils.getFloatValue(actual); + switch (builtinFunctionName) { + case ADD: + assertEquals(vf1 + vf2, vfActual); + return; + case SUBTRACT: + assertEquals(vf1 - vf2, vfActual); + return; + case DIVIDE: + assertEquals(vf1 / vf2, vfActual); + return; + case MULTIPLY: + assertEquals(vf1 * vf2, vfActual); + return; + case MODULES: + assertEquals(vf1 % vf2, vfActual); + return; + default: + throw new IllegalStateException("illegal function name " + builtinFunctionName); + } + case DOUBLE: + Double vd1 = ExprValueUtils.getDoubleValue(op1); + Double vd2 = ExprValueUtils.getDoubleValue(op2); + Double vdActual = ExprValueUtils.getDoubleValue(actual); + switch (builtinFunctionName) { + case ADD: + assertEquals(vd1 + vd2, vdActual); + return; + case SUBTRACT: + assertEquals(vd1 - vd2, vdActual); + return; + case DIVIDE: + assertEquals(vd1 / vd2, vdActual); + return; + case MULTIPLY: + assertEquals(vd1 * vd2, vdActual); + return; + case MODULES: + assertEquals(vd1 % vd2, vdActual); + return; + default: + throw new IllegalStateException("illegal function name " + builtinFunctionName); + } + default: + throw new IllegalStateException("illegal function name " + builtinFunctionName); + } + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/UnaryFunctionTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/UnaryFunctionTest.java new file mode 100644 index 0000000000..fca5ccdbd8 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/UnaryFunctionTest.java @@ -0,0 +1,103 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.operator.arthmetic; + +import static com.amazon.opendistroforelasticsearch.sql.config.TestConfig.INT_TYPE_MISSING_VALUE_FIELD; +import static com.amazon.opendistroforelasticsearch.sql.config.TestConfig.INT_TYPE_NULL_VALUE_FIELD; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.utils.MatcherUtils.hasType; +import static com.amazon.opendistroforelasticsearch.sql.utils.MatcherUtils.hasValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.allOf; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.amazon.opendistroforelasticsearch.sql.expression.ExpressionTestBase; +import com.amazon.opendistroforelasticsearch.sql.expression.FunctionExpression; +import org.junit.jupiter.api.DisplayNameGeneration; +import org.junit.jupiter.api.DisplayNameGenerator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +@DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) +public class UnaryFunctionTest extends ExpressionTestBase { + + /** + * Test abs with integer value. + */ + @ParameterizedTest(name = "abs({0})") + @ValueSource(ints = {-2, 2}) + public void abs_int_value(Integer value) { + FunctionExpression abs = dsl.abs(DSL.literal(value)); + assertThat( + abs.valueOf(valueEnv()), + allOf(hasType(INTEGER), hasValue(Math.abs(value)))); + assertEquals(String.format("abs(%s)", value.toString()), abs.toString()); + } + + /** + * Test abs with long value. + */ + @ParameterizedTest(name = "abs({0})") + @ValueSource(longs = {-2L, 2L}) + public void abs_long_value(Long value) { + FunctionExpression abs = dsl.abs(DSL.literal(value)); + assertThat( + abs.valueOf(valueEnv()), + allOf(hasType(ExprCoreType.LONG), hasValue(Math.abs(value)))); + assertEquals(String.format("abs(%s)", value.toString()), abs.toString()); + } + + /** + * Test abs with float value. + */ + @ParameterizedTest(name = "abs({0})") + @ValueSource(floats = {-2f, 2f}) + public void abs_float_value(Float value) { + FunctionExpression abs = dsl.abs(DSL.literal(value)); + assertThat( + abs.valueOf(valueEnv()), + allOf(hasType(ExprCoreType.FLOAT), hasValue(Math.abs(value)))); + assertEquals(String.format("abs(%s)", value.toString()), abs.toString()); + } + + /** + * Test abs with double value. + */ + @ParameterizedTest(name = "abs({0})") + @ValueSource(doubles = {-2L, 2L}) + public void abs_double_value(Double value) { + FunctionExpression abs = dsl.abs(DSL.literal(value)); + assertThat( + abs.valueOf(valueEnv()), + allOf(hasType(ExprCoreType.DOUBLE), hasValue(Math.abs(value)))); + assertEquals(String.format("abs(%s)", value.toString()), abs.toString()); + } + + @Test + public void abs_null_value() { + assertTrue(dsl.abs(DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER)).valueOf(valueEnv()).isNull()); + } + + @Test + public void abs_missing_value() { + assertTrue( + dsl.abs(DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER)).valueOf(valueEnv()).isMissing()); + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java new file mode 100644 index 0000000000..8da8967f30 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java @@ -0,0 +1,722 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.operator.predicate; + +import static com.amazon.opendistroforelasticsearch.sql.config.TestConfig.BOOL_TYPE_MISSING_VALUE_FIELD; +import static com.amazon.opendistroforelasticsearch.sql.config.TestConfig.BOOL_TYPE_NULL_VALUE_FIELD; +import static com.amazon.opendistroforelasticsearch.sql.config.TestConfig.INT_TYPE_MISSING_VALUE_FIELD; +import static com.amazon.opendistroforelasticsearch.sql.config.TestConfig.INT_TYPE_NULL_VALUE_FIELD; +import static com.amazon.opendistroforelasticsearch.sql.config.TestConfig.STRING_TYPE_MISSING_VALUE_FILED; +import static com.amazon.opendistroforelasticsearch.sql.config.TestConfig.STRING_TYPE_NULL_VALUE_FILED; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.booleanValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.fromObjectValue; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BOOLEAN; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.utils.ComparisonUtil.compare; +import static com.amazon.opendistroforelasticsearch.sql.utils.OperatorUtils.matches; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.amazon.opendistroforelasticsearch.sql.expression.ExpressionTestBase; +import com.amazon.opendistroforelasticsearch.sql.expression.FunctionExpression; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class BinaryPredicateOperatorTest extends ExpressionTestBase { + + private static Stream binaryPredicateArguments() { + List booleans = Arrays.asList(true, false); + return Lists.cartesianProduct(booleans, booleans).stream() + .map(list -> Arguments.of(list.get(0), list.get(1))); + } + + private static Stream testEqualArguments() { + List arguments = Arrays.asList(1, 1L, 1F, 1D, "str", true, ImmutableList.of(1), + ImmutableMap.of("str", 1)); + Stream.Builder builder = Stream.builder(); + for (Object argument : arguments) { + builder.add(Arguments.of(fromObjectValue(argument), fromObjectValue(argument))); + } + return builder.build(); + } + + private static Stream testNotEqualArguments() { + List> arguments = Arrays.asList( + Arrays.asList(1, 2), Arrays.asList(1L, 2L), Arrays.asList(1F, 2F), Arrays.asList(1D, 2D), + Arrays.asList("str0", "str1"), Arrays.asList(true, false), + Arrays.asList(ImmutableList.of(1), ImmutableList.of(2)), + Arrays.asList(ImmutableMap.of("str", 1), ImmutableMap.of("str", 2)) + ); + Stream.Builder builder = Stream.builder(); + for (List argPair : arguments) { + builder.add(Arguments.of(fromObjectValue(argPair.get(0)), fromObjectValue(argPair.get(1)))); + } + return builder.build(); + } + + private static Stream testCompareValueArguments() { + List> arguments = Arrays.asList( + Arrays.asList(1, 1), Arrays.asList(1, 2), Arrays.asList(2, 1), + Arrays.asList(1L, 1L), Arrays.asList(1L, 2L), Arrays.asList(2L, 1L), + Arrays.asList(1F, 1F), Arrays.asList(1F, 2F), Arrays.asList(2F, 1F), + Arrays.asList(1D, 1D), Arrays.asList(1D, 2D), Arrays.asList(2D, 1D), + Arrays.asList("str", "str"), Arrays.asList("str", "str0"), Arrays.asList("str0", "str") + ); + Stream.Builder builder = Stream.builder(); + for (List argPair : arguments) { + builder.add(Arguments.of(fromObjectValue(argPair.get(0)), fromObjectValue(argPair.get(1)))); + } + return builder.build(); + } + + private static Stream testLikeArguments() { + List> arguments = Arrays.asList( + Arrays.asList("foo", "foo"), Arrays.asList("notFoo", "foo"), + Arrays.asList("foobar", "%bar"), Arrays.asList("bar", "%bar"), + Arrays.asList("foo", "fo_"), Arrays.asList("foo", "foo_"), + Arrays.asList("foorbar", "%o_ar"), Arrays.asList("foobar", "%o_a%"), + Arrays.asList("fooba%_\\^$.*[]()|+r", "%\\%\\_\\\\\\^\\$\\.\\*\\[\\]\\(\\)\\|\\+_") + ); + Stream.Builder builder = Stream.builder(); + for (List argPair : arguments) { + builder.add(Arguments.of(fromObjectValue(argPair.get(0)), fromObjectValue(argPair.get(1)))); + } + return builder.build(); + } + + @ParameterizedTest(name = "and({0}, {1})") + @MethodSource("binaryPredicateArguments") + public void test_and(Boolean v1, Boolean v2) { + FunctionExpression and = + dsl.and(DSL.literal(booleanValue(v1)), DSL.literal(booleanValue(v2))); + assertEquals(BOOLEAN, and.type()); + assertEquals(v1 && v2, ExprValueUtils.getBooleanValue(and.valueOf(valueEnv()))); + assertEquals(String.format("%s and %s", v1.toString(), v2.toString()), and.toString()); + } + + @Test + public void test_boolean_and_null() { + FunctionExpression and = + dsl.and(DSL.literal(LITERAL_TRUE), DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, and.type()); + assertEquals(LITERAL_NULL, and.valueOf(valueEnv())); + + and = dsl.and(DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN), DSL.literal(LITERAL_TRUE)); + assertEquals(BOOLEAN, and.type()); + assertEquals(LITERAL_NULL, and.valueOf(valueEnv())); + + and = dsl.and(DSL.literal(LITERAL_FALSE), DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, and.type()); + assertEquals(LITERAL_FALSE, and.valueOf(valueEnv())); + + and = dsl.and(DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN), DSL.literal(LITERAL_FALSE)); + assertEquals(BOOLEAN, and.type()); + assertEquals(LITERAL_FALSE, and.valueOf(valueEnv())); + } + + @Test + public void test_boolean_and_missing() { + FunctionExpression and = + dsl.and(DSL.literal(LITERAL_TRUE), DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, and.type()); + assertEquals(LITERAL_MISSING, and.valueOf(valueEnv())); + + and = dsl.and(DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN), DSL.literal(LITERAL_TRUE)); + assertEquals(BOOLEAN, and.type()); + assertEquals(LITERAL_MISSING, and.valueOf(valueEnv())); + + and = dsl.and(DSL.literal(LITERAL_FALSE), DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, and.type()); + assertEquals(LITERAL_FALSE, and.valueOf(valueEnv())); + + and = dsl.and(DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN), DSL.literal(LITERAL_FALSE)); + assertEquals(BOOLEAN, and.type()); + assertEquals(LITERAL_FALSE, and.valueOf(valueEnv())); + } + + @Test + public void test_null_and_missing() { + FunctionExpression and = dsl.and(DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN), + DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, and.type()); + assertEquals(LITERAL_MISSING, and.valueOf(valueEnv())); + + and = dsl.and(DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN), + DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, and.type()); + assertEquals(LITERAL_NULL, and.valueOf(valueEnv())); + + and = dsl.and(DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN), + DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, and.type()); + assertEquals(LITERAL_MISSING, and.valueOf(valueEnv())); + + and = dsl.and(DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN), + DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, and.type()); + assertEquals(LITERAL_MISSING, and.valueOf(valueEnv())); + } + + @ParameterizedTest(name = "or({0}, {1})") + @MethodSource("binaryPredicateArguments") + public void test_or(Boolean v1, Boolean v2) { + FunctionExpression or = + dsl.or(DSL.literal(booleanValue(v1)), DSL.literal(booleanValue(v2))); + assertEquals(BOOLEAN, or.type()); + assertEquals(v1 || v2, ExprValueUtils.getBooleanValue(or.valueOf(valueEnv()))); + assertEquals(String.format("%s or %s", v1.toString(), v2.toString()), or.toString()); + } + + @Test + public void test_boolean_or_null() { + FunctionExpression or = + dsl.or(DSL.literal(LITERAL_TRUE), DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, or.type()); + assertEquals(LITERAL_TRUE, or.valueOf(valueEnv())); + + or = dsl.or(DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN), DSL.literal(LITERAL_TRUE)); + assertEquals(BOOLEAN, or.type()); + assertEquals(LITERAL_TRUE, or.valueOf(valueEnv())); + + or = dsl.or(DSL.literal(LITERAL_FALSE), DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, or.type()); + assertEquals(LITERAL_NULL, or.valueOf(valueEnv())); + + or = dsl.or(DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN), DSL.literal(LITERAL_FALSE)); + assertEquals(BOOLEAN, or.type()); + assertEquals(LITERAL_NULL, or.valueOf(valueEnv())); + } + + @Test + public void test_boolean_or_missing() { + FunctionExpression or = + dsl.or(DSL.literal(LITERAL_TRUE), DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, or.type()); + assertEquals(LITERAL_TRUE, or.valueOf(valueEnv())); + + or = dsl.or(DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN), DSL.literal(LITERAL_TRUE)); + assertEquals(BOOLEAN, or.type()); + assertEquals(LITERAL_TRUE, or.valueOf(valueEnv())); + + or = dsl.or(DSL.literal(LITERAL_FALSE), DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, or.type()); + assertEquals(LITERAL_MISSING, or.valueOf(valueEnv())); + + or = dsl.or(DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN), DSL.literal(LITERAL_FALSE)); + assertEquals(BOOLEAN, or.type()); + assertEquals(LITERAL_MISSING, or.valueOf(valueEnv())); + } + + @Test + public void test_null_or_missing() { + FunctionExpression or = dsl.or(DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN), + DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, or.type()); + assertEquals(LITERAL_MISSING, or.valueOf(valueEnv())); + + or = + dsl.or(DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN), + DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, or.type()); + assertEquals(LITERAL_NULL, or.valueOf(valueEnv())); + + or = dsl.or(DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN), + DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, or.type()); + assertEquals(LITERAL_NULL, or.valueOf(valueEnv())); + + or = dsl.or(DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN), + DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, or.type()); + assertEquals(LITERAL_NULL, or.valueOf(valueEnv())); + } + + + @ParameterizedTest(name = "xor({0}, {1})") + @MethodSource("binaryPredicateArguments") + public void test_xor(Boolean v1, Boolean v2) { + FunctionExpression xor = + dsl.xor(DSL.literal(booleanValue(v1)), DSL.literal(booleanValue(v2))); + assertEquals(BOOLEAN, xor.type()); + assertEquals(v1 ^ v2, ExprValueUtils.getBooleanValue(xor.valueOf(valueEnv()))); + assertEquals(String.format("%s xor %s", v1.toString(), v2.toString()), xor.toString()); + } + + @Test + public void test_boolean_xor_null() { + FunctionExpression xor = + dsl.xor(DSL.literal(LITERAL_TRUE), DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, xor.type()); + assertEquals(LITERAL_TRUE, xor.valueOf(valueEnv())); + + xor = dsl.xor(DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN), DSL.literal(LITERAL_TRUE)); + assertEquals(BOOLEAN, xor.type()); + assertEquals(LITERAL_TRUE, xor.valueOf(valueEnv())); + + xor = dsl.xor(DSL.literal(LITERAL_FALSE), DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, xor.type()); + assertEquals(LITERAL_NULL, xor.valueOf(valueEnv())); + + xor = dsl.xor(DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN), DSL.literal(LITERAL_FALSE)); + assertEquals(BOOLEAN, xor.type()); + assertEquals(LITERAL_NULL, xor.valueOf(valueEnv())); + } + + @Test + public void test_boolean_xor_missing() { + FunctionExpression xor = + dsl.xor(DSL.literal(LITERAL_TRUE), DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, xor.type()); + assertEquals(LITERAL_TRUE, xor.valueOf(valueEnv())); + + xor = dsl.xor(DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN), DSL.literal(LITERAL_TRUE)); + assertEquals(BOOLEAN, xor.type()); + assertEquals(LITERAL_TRUE, xor.valueOf(valueEnv())); + + xor = dsl.xor(DSL.literal(LITERAL_FALSE), DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, xor.type()); + assertEquals(LITERAL_MISSING, xor.valueOf(valueEnv())); + + xor = dsl.xor(DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN), DSL.literal(LITERAL_FALSE)); + assertEquals(BOOLEAN, xor.type()); + assertEquals(LITERAL_MISSING, xor.valueOf(valueEnv())); + } + + @Test + public void test_null_xor_missing() { + FunctionExpression xor = dsl.xor(DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN), + DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, xor.type()); + assertEquals(LITERAL_MISSING, xor.valueOf(valueEnv())); + + xor = dsl.xor(DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN), + DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, xor.type()); + assertEquals(LITERAL_NULL, xor.valueOf(valueEnv())); + + xor = dsl.xor(DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN), + DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, xor.type()); + assertEquals(LITERAL_NULL, xor.valueOf(valueEnv())); + + xor = dsl.xor(DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN), + DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, xor.type()); + assertEquals(LITERAL_NULL, xor.valueOf(valueEnv())); + } + + @ParameterizedTest(name = "equal({0}, {1})") + @MethodSource("testEqualArguments") + public void test_equal(ExprValue v1, ExprValue v2) { + FunctionExpression equal = dsl.equal(DSL.literal(v1), DSL.literal(v2)); + assertEquals(BOOLEAN, equal.type()); + assertEquals(v1.value().equals(v2.value()), + ExprValueUtils.getBooleanValue(equal.valueOf(valueEnv()))); + assertEquals(String.format("%s = %s", v1.toString(), v2.toString()), equal.toString()); + } + + @Test + public void test_null_equal_missing() { + FunctionExpression equal = dsl.equal(DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN), + DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, equal.type()); + assertEquals(LITERAL_TRUE, equal.valueOf(valueEnv())); + + equal = dsl.equal(DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN), + DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, equal.type()); + assertEquals(LITERAL_TRUE, equal.valueOf(valueEnv())); + + equal = dsl.equal(DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN), + DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, equal.type()); + assertEquals(LITERAL_FALSE, equal.valueOf(valueEnv())); + + equal = dsl.equal(DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN), + DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, equal.type()); + assertEquals(LITERAL_FALSE, equal.valueOf(valueEnv())); + + equal = dsl.equal(DSL.literal(LITERAL_TRUE), DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, equal.type()); + assertEquals(LITERAL_FALSE, equal.valueOf(valueEnv())); + + equal = dsl.equal(DSL.literal(LITERAL_TRUE), DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, equal.type()); + assertEquals(LITERAL_FALSE, equal.valueOf(valueEnv())); + + equal = dsl.equal(DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN), DSL.literal(LITERAL_TRUE)); + assertEquals(BOOLEAN, equal.type()); + assertEquals(LITERAL_FALSE, equal.valueOf(valueEnv())); + + equal = dsl.equal(DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN), DSL.literal(LITERAL_TRUE)); + assertEquals(BOOLEAN, equal.type()); + assertEquals(LITERAL_FALSE, equal.valueOf(valueEnv())); + } + + @ParameterizedTest(name = "equal({0}, {1})") + @MethodSource({"testEqualArguments", "testNotEqualArguments"}) + public void test_notequal(ExprValue v1, ExprValue v2) { + FunctionExpression notequal = dsl.notequal(DSL.literal(v1), DSL.literal(v2)); + assertEquals(BOOLEAN, notequal.type()); + assertEquals(!v1.value().equals(v2.value()), + ExprValueUtils.getBooleanValue(notequal.valueOf(valueEnv()))); + assertEquals(String.format("%s != %s", v1.toString(), v2.toString()), notequal.toString()); + } + + @Test + public void test_null_notequal_missing() { + FunctionExpression notequal = dsl.notequal(DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN), + DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, notequal.type()); + assertEquals(LITERAL_FALSE, notequal.valueOf(valueEnv())); + + notequal = dsl.notequal(DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN), + DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, notequal.type()); + assertEquals(LITERAL_FALSE, notequal.valueOf(valueEnv())); + + notequal = dsl.notequal(DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN), + DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, notequal.type()); + assertEquals(LITERAL_TRUE, notequal.valueOf(valueEnv())); + + notequal = dsl.notequal(DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN), + DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, notequal.type()); + assertEquals(LITERAL_TRUE, notequal.valueOf(valueEnv())); + + notequal = dsl.notequal(DSL.literal(LITERAL_TRUE), + DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, notequal.type()); + assertEquals(LITERAL_TRUE, notequal.valueOf(valueEnv())); + + notequal = dsl.notequal(DSL.literal(LITERAL_TRUE), + DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, notequal.type()); + assertEquals(LITERAL_TRUE, notequal.valueOf(valueEnv())); + + notequal = dsl.notequal(DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN), + DSL.literal(LITERAL_TRUE)); + assertEquals(BOOLEAN, notequal.type()); + assertEquals(LITERAL_TRUE, notequal.valueOf(valueEnv())); + + notequal = dsl.notequal(DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN), + DSL.literal(LITERAL_TRUE)); + assertEquals(BOOLEAN, notequal.type()); + assertEquals(LITERAL_TRUE, notequal.valueOf(valueEnv())); + } + + @ParameterizedTest(name = "less({0}, {1})") + @MethodSource("testCompareValueArguments") + public void test_less(ExprValue v1, ExprValue v2) { + FunctionExpression less = dsl.less(DSL.literal(v1), DSL.literal(v2)); + assertEquals(BOOLEAN, less.type()); + assertEquals(compare(v1, v2) < 0, + ExprValueUtils.getBooleanValue(less.valueOf(valueEnv()))); + assertEquals(String.format("%s < %s", v1.toString(), v2.toString()), less.toString()); + } + + @Test + public void test_less_null() { + FunctionExpression less = dsl.less(DSL.literal(1), + DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER)); + assertEquals(BOOLEAN, less.type()); + assertEquals(LITERAL_NULL, less.valueOf(valueEnv())); + + less = dsl.less(DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER), DSL.literal(1)); + assertEquals(BOOLEAN, less.type()); + assertEquals(LITERAL_NULL, less.valueOf(valueEnv())); + + less = dsl.less(DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER), + DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER)); + assertEquals(BOOLEAN, less.type()); + assertEquals(LITERAL_NULL, less.valueOf(valueEnv())); + } + + @Test + public void test_less_missing() { + FunctionExpression less = dsl.less(DSL.literal(1), + DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER)); + assertEquals(BOOLEAN, less.type()); + assertEquals(LITERAL_MISSING, less.valueOf(valueEnv())); + + less = dsl.less(DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER), DSL.literal(1)); + assertEquals(BOOLEAN, less.type()); + assertEquals(LITERAL_MISSING, less.valueOf(valueEnv())); + + less = dsl.less(DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER), + DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER)); + assertEquals(BOOLEAN, less.type()); + assertEquals(LITERAL_MISSING, less.valueOf(valueEnv())); + } + + @Test + public void test_null_less_missing() { + FunctionExpression less = dsl.less(DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER), + DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER)); + assertEquals(BOOLEAN, less.type()); + assertEquals(LITERAL_MISSING, less.valueOf(valueEnv())); + + less = dsl.less(DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER), + DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER)); + assertEquals(BOOLEAN, less.type()); + assertEquals(LITERAL_MISSING, less.valueOf(valueEnv())); + } + + @ParameterizedTest(name = "lte({0}, {1})") + @MethodSource("testCompareValueArguments") + public void test_lte(ExprValue v1, ExprValue v2) { + FunctionExpression lte = dsl.lte(DSL.literal(v1), DSL.literal(v2)); + assertEquals(BOOLEAN, lte.type()); + assertEquals(compare(v1, v2) <= 0, + ExprValueUtils.getBooleanValue(lte.valueOf(valueEnv()))); + assertEquals(String.format("%s <= %s", v1.toString(), v2.toString()), lte.toString()); + } + + @Test + public void test_lte_null() { + FunctionExpression lte = dsl.lte(DSL.literal(1), + DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER)); + assertEquals(BOOLEAN, lte.type()); + assertEquals(LITERAL_NULL, lte.valueOf(valueEnv())); + + lte = dsl.lte(DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER), DSL.literal(1)); + assertEquals(BOOLEAN, lte.type()); + assertEquals(LITERAL_NULL, lte.valueOf(valueEnv())); + + lte = dsl.lte(DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER), + DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER)); + assertEquals(BOOLEAN, lte.type()); + assertEquals(LITERAL_NULL, lte.valueOf(valueEnv())); + } + + @Test + public void test_lte_missing() { + FunctionExpression lte = dsl.lte(DSL.literal(1), + DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER)); + assertEquals(BOOLEAN, lte.type()); + assertEquals(LITERAL_MISSING, lte.valueOf(valueEnv())); + + lte = dsl.lte(DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER), DSL.literal(1)); + assertEquals(BOOLEAN, lte.type()); + assertEquals(LITERAL_MISSING, lte.valueOf(valueEnv())); + + lte = dsl.lte(DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER), + DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER)); + assertEquals(BOOLEAN, lte.type()); + assertEquals(LITERAL_MISSING, lte.valueOf(valueEnv())); + } + + @Test + public void test_null_lte_missing() { + FunctionExpression lte = dsl.lte(DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER), + DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER)); + assertEquals(BOOLEAN, lte.type()); + assertEquals(LITERAL_MISSING, lte.valueOf(valueEnv())); + + lte = dsl.lte(DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER), + DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER)); + assertEquals(BOOLEAN, lte.type()); + assertEquals(LITERAL_MISSING, lte.valueOf(valueEnv())); + } + + @ParameterizedTest(name = "greater({0}, {1})") + @MethodSource("testCompareValueArguments") + public void test_greater(ExprValue v1, ExprValue v2) { + FunctionExpression greater = dsl.greater(DSL.literal(v1), DSL.literal(v2)); + assertEquals(BOOLEAN, greater.type()); + assertEquals(compare(v1, v2) > 0, + ExprValueUtils.getBooleanValue(greater.valueOf(valueEnv()))); + assertEquals(String.format("%s > %s", v1.toString(), v2.toString()), greater.toString()); + } + + @Test + public void test_greater_null() { + FunctionExpression greater = dsl.greater(DSL.literal(1), + DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER)); + assertEquals(BOOLEAN, greater.type()); + assertEquals(LITERAL_NULL, greater.valueOf(valueEnv())); + + greater = dsl.greater(DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER), DSL.literal(1)); + assertEquals(BOOLEAN, greater.type()); + assertEquals(LITERAL_NULL, greater.valueOf(valueEnv())); + + greater = dsl.greater(DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER), + DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER)); + assertEquals(BOOLEAN, greater.type()); + assertEquals(LITERAL_NULL, greater.valueOf(valueEnv())); + } + + @Test + public void test_greater_missing() { + FunctionExpression greater = dsl.greater(DSL.literal(1), + DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER)); + assertEquals(BOOLEAN, greater.type()); + assertEquals(LITERAL_MISSING, greater.valueOf(valueEnv())); + + greater = dsl.greater(DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER), DSL.literal(1)); + assertEquals(BOOLEAN, greater.type()); + assertEquals(LITERAL_MISSING, greater.valueOf(valueEnv())); + + greater = dsl.greater(DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER), + DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER)); + assertEquals(BOOLEAN, greater.type()); + assertEquals(LITERAL_MISSING, greater.valueOf(valueEnv())); + } + + @Test + public void test_null_greater_missing() { + FunctionExpression greater = dsl.greater(DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER), + DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER)); + assertEquals(BOOLEAN, greater.type()); + assertEquals(LITERAL_MISSING, greater.valueOf(valueEnv())); + + greater = dsl.greater(DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER), + DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER)); + assertEquals(BOOLEAN, greater.type()); + assertEquals(LITERAL_MISSING, greater.valueOf(valueEnv())); + } + + @ParameterizedTest(name = "gte({0}, {1})") + @MethodSource("testCompareValueArguments") + public void test_gte(ExprValue v1, ExprValue v2) { + FunctionExpression gte = dsl.gte(DSL.literal(v1), DSL.literal(v2)); + assertEquals(BOOLEAN, gte.type()); + assertEquals(compare(v1, v2) >= 0, + ExprValueUtils.getBooleanValue(gte.valueOf(valueEnv()))); + assertEquals(String.format("%s >= %s", v1.toString(), v2.toString()), gte.toString()); + } + + @Test + public void test_gte_null() { + FunctionExpression gte = dsl.gte(DSL.literal(1), + DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER)); + assertEquals(BOOLEAN, gte.type()); + assertEquals(LITERAL_NULL, gte.valueOf(valueEnv())); + + gte = dsl.gte(DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER), DSL.literal(1)); + assertEquals(BOOLEAN, gte.type()); + assertEquals(LITERAL_NULL, gte.valueOf(valueEnv())); + + gte = dsl.gte(DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER), + DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER)); + assertEquals(BOOLEAN, gte.type()); + assertEquals(LITERAL_NULL, gte.valueOf(valueEnv())); + } + + @Test + public void test_gte_missing() { + FunctionExpression gte = dsl.gte(DSL.literal(1), + DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER)); + assertEquals(BOOLEAN, gte.type()); + assertEquals(LITERAL_MISSING, gte.valueOf(valueEnv())); + + gte = dsl.gte(DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER), DSL.literal(1)); + assertEquals(BOOLEAN, gte.type()); + assertEquals(LITERAL_MISSING, gte.valueOf(valueEnv())); + + gte = dsl.gte(DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER), + DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER)); + assertEquals(BOOLEAN, gte.type()); + assertEquals(LITERAL_MISSING, gte.valueOf(valueEnv())); + } + + @Test + public void test_null_gte_missing() { + FunctionExpression gte = dsl.gte(DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER), + DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER)); + assertEquals(BOOLEAN, gte.type()); + assertEquals(LITERAL_MISSING, gte.valueOf(valueEnv())); + + gte = dsl.gte(DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER), + DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER)); + assertEquals(BOOLEAN, gte.type()); + assertEquals(LITERAL_MISSING, gte.valueOf(valueEnv())); + } + + @ParameterizedTest(name = "like({0}, {1})") + @MethodSource("testLikeArguments") + public void test_like(ExprValue v1, ExprValue v2) { + FunctionExpression like = dsl.like(DSL.literal(v1), DSL.literal(v2)); + assertEquals(BOOLEAN, like.type()); + assertEquals(matches(((String) v2.value()), (String) v1.value()), + ExprValueUtils.getBooleanValue(like.valueOf(valueEnv()))); + assertEquals(String.format("%s like %s", v1.toString(), v2.toString()), like.toString()); + } + + @Test + public void test_like_null() { + FunctionExpression like = + dsl.like(DSL.literal("str"), DSL.ref(STRING_TYPE_NULL_VALUE_FILED, STRING)); + assertEquals(BOOLEAN, like.type()); + assertEquals(LITERAL_NULL, like.valueOf(valueEnv())); + + like = dsl.like(DSL.ref(STRING_TYPE_NULL_VALUE_FILED, STRING), DSL.literal("str")); + assertEquals(BOOLEAN, like.type()); + assertEquals(LITERAL_NULL, like.valueOf(valueEnv())); + + like = dsl.like(DSL.ref(STRING_TYPE_NULL_VALUE_FILED, STRING), + DSL.ref(STRING_TYPE_NULL_VALUE_FILED, STRING)); + assertEquals(BOOLEAN, like.type()); + assertEquals(LITERAL_NULL, like.valueOf(valueEnv())); + } + + @Test + public void test_like_missing() { + FunctionExpression like = + dsl.like(DSL.literal("str"), DSL.ref(STRING_TYPE_MISSING_VALUE_FILED, STRING)); + assertEquals(BOOLEAN, like.type()); + assertEquals(LITERAL_MISSING, like.valueOf(valueEnv())); + + like = dsl.like(DSL.ref(STRING_TYPE_MISSING_VALUE_FILED, STRING), DSL.literal("str")); + assertEquals(BOOLEAN, like.type()); + assertEquals(LITERAL_MISSING, like.valueOf(valueEnv())); + + like = dsl.like(DSL.ref(STRING_TYPE_MISSING_VALUE_FILED, STRING), + DSL.ref(STRING_TYPE_MISSING_VALUE_FILED, STRING)); + assertEquals(BOOLEAN, like.type()); + assertEquals(LITERAL_MISSING, like.valueOf(valueEnv())); + } + + @Test + public void test_null_like_missing() { + FunctionExpression like = dsl.like(DSL.ref(STRING_TYPE_NULL_VALUE_FILED, STRING), + DSL.ref(STRING_TYPE_MISSING_VALUE_FILED, STRING)); + assertEquals(BOOLEAN, like.type()); + assertEquals(LITERAL_MISSING, like.valueOf(valueEnv())); + + like = dsl.like(DSL.ref(STRING_TYPE_MISSING_VALUE_FILED, STRING), + DSL.ref(STRING_TYPE_NULL_VALUE_FILED, STRING)); + assertEquals(BOOLEAN, like.type()); + assertEquals(LITERAL_MISSING, like.valueOf(valueEnv())); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/UnaryPredicateOperatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/UnaryPredicateOperatorTest.java new file mode 100644 index 0000000000..374f8e0fc8 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/UnaryPredicateOperatorTest.java @@ -0,0 +1,57 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.operator.predicate; + +import static com.amazon.opendistroforelasticsearch.sql.config.TestConfig.BOOL_TYPE_MISSING_VALUE_FIELD; +import static com.amazon.opendistroforelasticsearch.sql.config.TestConfig.BOOL_TYPE_NULL_VALUE_FIELD; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.booleanValue; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BOOLEAN; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.amazon.opendistroforelasticsearch.sql.expression.ExpressionTestBase; +import com.amazon.opendistroforelasticsearch.sql.expression.FunctionExpression; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class UnaryPredicateOperatorTest extends ExpressionTestBase { + @ParameterizedTest(name = "not({0})") + @ValueSource(booleans = {true, false}) + public void test_not(Boolean v) { + FunctionExpression not = dsl.not(DSL.literal(booleanValue(v))); + assertEquals(BOOLEAN, not.type()); + assertEquals(!v, ExprValueUtils.getBooleanValue(not.valueOf(valueEnv()))); + assertEquals(String.format("not %s", v.toString()), not.toString()); + } + + @Test + public void test_not_null() { + FunctionExpression and = dsl.not(DSL.ref(BOOL_TYPE_NULL_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, and.type()); + assertEquals(LITERAL_NULL, and.valueOf(valueEnv())); + } + + @Test + public void test_not_missing() { + FunctionExpression and = dsl.not(DSL.ref(BOOL_TYPE_MISSING_VALUE_FIELD, BOOLEAN)); + assertEquals(BOOLEAN, and.type()); + assertEquals(LITERAL_MISSING, and.valueOf(valueEnv())); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/monitor/AlwaysHealthyMonitorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/monitor/AlwaysHealthyMonitorTest.java new file mode 100644 index 0000000000..6e83cc54e6 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/monitor/AlwaysHealthyMonitorTest.java @@ -0,0 +1,30 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.monitor; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class AlwaysHealthyMonitorTest { + + @Test + void isHealthy() { + assertTrue(new AlwaysHealthyMonitor().isHealthy()); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementorTest.java new file mode 100644 index 0000000000..ce0495c97b --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementorTest.java @@ -0,0 +1,128 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.planner; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.literal; +import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.ref; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.aggregation; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.eval; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.filter; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.project; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.remove; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.rename; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.sort; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.values; +import static java.util.Collections.emptyList; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; +import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.AvgAggregator; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRelation; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlanDSL; +import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.Test; + +class DefaultImplementorTest { + + private final DefaultImplementor implementor = new DefaultImplementor<>(); + + @Test + public void visitShouldReturnDefaultPhysicalOperator() { + String indexName = "test"; + ReferenceExpression include = ref("age", INTEGER); + ReferenceExpression exclude = ref("name", STRING); + ReferenceExpression dedupeField = ref("name", STRING); + Expression filterExpr = literal(ExprBooleanValue.ofTrue()); + List groupByExprs = Arrays.asList(ref("age", INTEGER)); + List aggregators = + Arrays.asList(new AvgAggregator(groupByExprs, ExprCoreType.DOUBLE)); + Map mappings = + ImmutableMap.of(ref("name", STRING), ref("lastname", STRING)); + Pair newEvalField = + ImmutablePair.of(ref("name1", STRING), ref("name", STRING)); + Integer sortCount = 100; + Pair sortField = + ImmutablePair.of(Sort.SortOption.PPL_ASC, ref("name1", STRING)); + + LogicalPlan plan = + project( + LogicalPlanDSL.dedupe( + sort( + eval( + remove( + rename( + aggregation( + filter(values(emptyList()), filterExpr), + aggregators, + groupByExprs), + mappings), + exclude), + newEvalField), + sortCount, + sortField), + dedupeField), + include); + + PhysicalPlan actual = plan.accept(implementor, null); + + assertEquals( + PhysicalPlanDSL.project( + PhysicalPlanDSL.dedupe( + PhysicalPlanDSL.sort( + PhysicalPlanDSL.eval( + PhysicalPlanDSL.remove( + PhysicalPlanDSL.rename( + PhysicalPlanDSL.agg( + PhysicalPlanDSL.filter( + PhysicalPlanDSL.values(emptyList()), + filterExpr), + aggregators, + groupByExprs), + mappings), + exclude), + newEvalField), + sortCount, + sortField), + dedupeField), + include), + actual); + + } + + @Test + public void visitRelationShouldThrowException() { + assertThrows(UnsupportedOperationException.class, + () -> new LogicalRelation("test").accept(implementor, null)); + } + +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/PlannerTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/PlannerTest.java new file mode 100644 index 0000000000..eddd35493e --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/PlannerTest.java @@ -0,0 +1,157 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static java.util.Collections.emptyList; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalAggregation; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalFilter; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanNodeVisitor; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRelation; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRename; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.AggregationOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.FilterOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlanDSL; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlanTestBase; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.RenameOperator; +import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; +import com.amazon.opendistroforelasticsearch.sql.storage.Table; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class PlannerTest extends PhysicalPlanTestBase { + @Mock + private PhysicalPlan scan; + + @Mock + private StorageEngine storageEngine; + + @BeforeEach + public void setUp() { + when(storageEngine.getTable(any())).thenReturn(new MockTable()); + } + + @Test + public void planner_test() { + assertPhysicalPlan( + PhysicalPlanDSL.rename( + PhysicalPlanDSL.agg( + PhysicalPlanDSL.filter( + scan, + dsl.equal(DSL.ref("response", INTEGER), DSL.literal(10)) + ), + ImmutableList.of(dsl.avg(DSL.ref("response", INTEGER))), + ImmutableList.of() + ), + ImmutableMap.of(DSL.ref("ivalue", INTEGER), DSL.ref("avg(response)", DOUBLE)) + ), + LogicalPlanDSL.rename( + LogicalPlanDSL.aggregation( + LogicalPlanDSL.filter( + LogicalPlanDSL.relation("schema"), + dsl.equal(DSL.ref("response", INTEGER), DSL.literal(10)) + ), + ImmutableList.of(dsl.avg(DSL.ref("response", INTEGER))), + ImmutableList.of() + ), + ImmutableMap.of(DSL.ref("ivalue", INTEGER), DSL.ref("avg(response)", DOUBLE)) + ) + ); + } + + @Test + public void plan_a_query_without_relation_involved() { + // Storage engine mock is not needed here since no relation involved. + Mockito.reset(storageEngine); + + assertPhysicalPlan( + PhysicalPlanDSL.project( + PhysicalPlanDSL.values(emptyList()), + DSL.literal(123), + DSL.literal("hello"), + DSL.literal(false) + ), + LogicalPlanDSL.project( + LogicalPlanDSL.values(emptyList()), + DSL.literal(123), + DSL.literal("hello"), + DSL.literal(false) + ) + ); + } + + protected void assertPhysicalPlan(PhysicalPlan expected, LogicalPlan logicalPlan) { + assertEquals(expected, analyze(logicalPlan)); + } + + protected PhysicalPlan analyze(LogicalPlan logicalPlan) { + return new Planner(storageEngine).plan(logicalPlan); + } + + protected class MockTable extends LogicalPlanNodeVisitor implements Table { + + @Override + public Map getFieldTypes() { + throw new UnsupportedOperationException(); + } + + @Override + public PhysicalPlan implement(LogicalPlan plan) { + return plan.accept(this, null); + } + + @Override + public PhysicalPlan visitRelation(LogicalRelation plan, Object context) { + return scan; + } + + @Override + public PhysicalPlan visitFilter(LogicalFilter plan, Object context) { + return new FilterOperator(plan.getChild().get(0).accept(this, context), plan.getCondition()); + } + + @Override + public PhysicalPlan visitAggregation(LogicalAggregation plan, Object context) { + return new AggregationOperator(plan.getChild().get(0).accept(this, context), + plan.getAggregatorList(), plan.getGroupByList() + ); + } + + @Override + public PhysicalPlan visitRename(LogicalRename plan, Object context) { + return new RenameOperator(plan.getChild().get(0).accept(this, context), + plan.getRenameMap()); + } + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalDedupeTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalDedupeTest.java new file mode 100644 index 0000000000..ffb8becb04 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalDedupeTest.java @@ -0,0 +1,66 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.argument; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.booleanLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.dedupe; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.defaultDedupArgs; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.exprList; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.field; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.intLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.relation; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; + +import com.amazon.opendistroforelasticsearch.sql.analysis.AnalyzerTestBase; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import org.junit.jupiter.api.Test; + +class LogicalDedupeTest extends AnalyzerTestBase { + @Test + public void analyze_dedup_with_two_field_with_default_option() { + assertAnalyzeEqual( + LogicalPlanDSL.dedupe( + LogicalPlanDSL.relation("schema"), + DSL.ref("integer_value", INTEGER), + DSL.ref("double_value", DOUBLE)), + dedupe( + relation("schema"), + defaultDedupArgs(), + field("integer_value"), field("double_value") + )); + } + + @Test + public void analyze_dedup_with_one_field_with_customize_option() { + assertAnalyzeEqual( + LogicalPlanDSL.dedupe( + LogicalPlanDSL.relation("schema"), + 3, false, true, + DSL.ref("integer_value", INTEGER), + DSL.ref("double_value", DOUBLE)), + dedupe( + relation("schema"), + exprList( + argument("number", intLiteral(3)), + argument("keepempty", booleanLiteral(false)), + argument("consecutive", booleanLiteral(true)) + ), + field("integer_value"), field("double_value") + )); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalEvalTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalEvalTest.java new file mode 100644 index 0000000000..cb985f8d6b --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalEvalTest.java @@ -0,0 +1,57 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.field; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; + +import com.amazon.opendistroforelasticsearch.sql.analysis.AnalyzerTestBase; +import com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class LogicalEvalTest extends AnalyzerTestBase { + + @Test + public void analyze_eval_with_one_field() { + assertAnalyzeEqual( + LogicalPlanDSL.eval( + LogicalPlanDSL.relation("schema"), + ImmutablePair + .of(DSL.ref("absValue", INTEGER), dsl.abs(DSL.ref("integer_value", INTEGER)))), + AstDSL.eval( + AstDSL.relation("schema"), + AstDSL.let(AstDSL.field("absValue"), AstDSL.function("abs", field("integer_value"))))); + } + + @Test + public void analyze_eval_with_two_field() { + assertAnalyzeEqual( + LogicalPlanDSL.eval( + LogicalPlanDSL.relation("schema"), + ImmutablePair + .of(DSL.ref("absValue", INTEGER), dsl.abs(DSL.ref("integer_value", INTEGER))), + ImmutablePair.of(DSL.ref("iValue", INTEGER), dsl.abs(DSL.ref("absValue", INTEGER)))), + AstDSL.eval( + AstDSL.relation("schema"), + AstDSL.let(AstDSL.field("absValue"), AstDSL.function("abs", field("integer_value"))), + AstDSL.let(AstDSL.field("iValue"), AstDSL.function("abs", field("absValue"))))); + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java new file mode 100644 index 0000000000..4f9dcb2107 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java @@ -0,0 +1,131 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort.SortOption; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.util.stream.Collectors; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +/** + * Todo. Temporary added for UT coverage, Will be removed. + */ +@ExtendWith(MockitoExtension.class) +class LogicalPlanNodeVisitorTest { + @Mock + Expression expression; + @Mock + ReferenceExpression ref; + @Mock + Aggregator aggregator; + + @Test + public void logicalPlanShouldTraversable() { + LogicalPlan logicalPlan = + LogicalPlanDSL.rename( + LogicalPlanDSL.aggregation( + LogicalPlanDSL.filter(LogicalPlanDSL.relation("schema"), expression), + ImmutableList.of(aggregator), + ImmutableList.of(expression)), + ImmutableMap.of(ref, ref)); + + Integer result = logicalPlan.accept(new NodesCount(), null); + assertEquals(4, result); + } + + @Test + public void testAbstractPlanNodeVisitorShouldReturnNull() { + LogicalPlan relation = LogicalPlanDSL.relation("schema"); + assertNull(relation.accept(new LogicalPlanNodeVisitor() { + }, null)); + + LogicalPlan filter = LogicalPlanDSL.filter(relation, expression); + assertNull(filter.accept(new LogicalPlanNodeVisitor() { + }, null)); + + LogicalPlan aggregation = + LogicalPlanDSL.aggregation( + filter, ImmutableList.of(aggregator), ImmutableList.of(expression)); + assertNull(aggregation.accept(new LogicalPlanNodeVisitor() { + }, null)); + + LogicalPlan rename = LogicalPlanDSL.rename(aggregation, ImmutableMap.of(ref, ref)); + assertNull(rename.accept(new LogicalPlanNodeVisitor() { + }, null)); + + LogicalPlan project = LogicalPlanDSL.project(relation, ref); + assertNull(project.accept(new LogicalPlanNodeVisitor() { + }, null)); + + LogicalPlan remove = LogicalPlanDSL.remove(relation, ref); + assertNull(remove.accept(new LogicalPlanNodeVisitor() { + }, null)); + + LogicalPlan eval = LogicalPlanDSL.eval(relation, Pair.of(ref, expression)); + assertNull(eval.accept(new LogicalPlanNodeVisitor() { + }, null)); + + LogicalPlan sort = LogicalPlanDSL.sort(relation, 100, Pair.of(SortOption.PPL_ASC, expression)); + assertNull(sort.accept(new LogicalPlanNodeVisitor() { + }, null)); + + LogicalPlan dedup = LogicalPlanDSL.dedupe(relation, 1, false, false, expression); + assertNull(dedup.accept(new LogicalPlanNodeVisitor() { + }, null)); + } + + private static class NodesCount extends LogicalPlanNodeVisitor { + @Override + public Integer visitRelation(LogicalRelation plan, Object context) { + return 1; + } + + @Override + public Integer visitFilter(LogicalFilter plan, Object context) { + return 1 + + plan.getChild().stream() + .map(child -> child.accept(this, context)) + .collect(Collectors.summingInt(Integer::intValue)); + } + + @Override + public Integer visitAggregation(LogicalAggregation plan, Object context) { + return 1 + + plan.getChild().stream() + .map(child -> child.accept(this, context)) + .collect(Collectors.summingInt(Integer::intValue)); + } + + @Override + public Integer visitRename(LogicalRename plan, Object context) { + return 1 + + plan.getChild().stream() + .map(child -> child.accept(this, context)) + .collect(Collectors.summingInt(Integer::intValue)); + } + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRelationTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRelationTest.java new file mode 100644 index 0000000000..543c7a01ac --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRelationTest.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class LogicalRelationTest { + + @Test + public void logicalRelationHasNoInput() { + LogicalPlan relation = LogicalPlanDSL.relation("index"); + assertEquals(0, relation.getChild().size()); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalSortTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalSortTest.java new file mode 100644 index 0000000000..a194fa62e6 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalSortTest.java @@ -0,0 +1,69 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.argument; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.booleanLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.defaultSortFieldArgs; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.defaultSortOptions; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.exprList; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.field; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.nullLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.relation; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.sort; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.sortOptions; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; + +import com.amazon.opendistroforelasticsearch.sql.analysis.AnalyzerTestBase; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort.SortOption; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.junit.jupiter.api.Test; + +class LogicalSortTest extends AnalyzerTestBase { + @Test + public void analyze_sort_with_two_field_with_default_option() { + assertAnalyzeEqual( + LogicalPlanDSL.sort( + LogicalPlanDSL.relation("schema"), + 1000, + ImmutablePair.of(SortOption.PPL_ASC, DSL.ref("integer_value", INTEGER)), + ImmutablePair.of(SortOption.PPL_ASC, DSL.ref("double_value", DOUBLE))), + sort( + relation("schema"), + defaultSortOptions(), + field("integer_value", defaultSortFieldArgs()), + field("double_value", defaultSortFieldArgs()))); + } + + @Test + public void analyze_sort_with_two_field() { + assertAnalyzeEqual( + LogicalPlanDSL.sort( + LogicalPlanDSL.relation("schema"), + 100, + ImmutablePair.of(SortOption.PPL_DESC, DSL.ref("integer_value", INTEGER)), + ImmutablePair.of(SortOption.PPL_ASC, DSL.ref("double_value", DOUBLE))), + sort( + relation("schema"), + sortOptions(100), + field( + "integer_value", + exprList(argument("asc", booleanLiteral(false)), argument("type", nullLiteral()))), + field("double_value", defaultSortFieldArgs()))); + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/AggregationOperatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/AggregationOperatorTest.java new file mode 100644 index 0000000000..fbfca109c9 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/AggregationOperatorTest.java @@ -0,0 +1,76 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.junit.jupiter.api.Test; + +class AggregationOperatorTest extends PhysicalPlanTestBase { + @Test + public void avg_with_one_groups() { + PhysicalPlan plan = new AggregationOperator(new TestScan(), + Collections.singletonList(dsl.avg(DSL.ref("response", INTEGER))), + Collections.singletonList(DSL.ref("action", STRING))); + List result = execute(plan); + assertEquals(2, result.size()); + assertThat(result, containsInAnyOrder( + ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET", "avg(response)", 268d)), + ExprValueUtils.tupleValue(ImmutableMap.of("action", "POST", "avg(response)", 350d)) + )); + } + + @Test + public void avg_with_two_groups() { + PhysicalPlan plan = new AggregationOperator(new TestScan(), + Collections.singletonList(dsl.avg(DSL.ref("response", INTEGER))), + Arrays.asList(DSL.ref("action", STRING), DSL.ref("ip", STRING))); + List result = execute(plan); + assertEquals(3, result.size()); + assertThat(result, containsInAnyOrder( + ExprValueUtils.tupleValue( + ImmutableMap.of("action", "GET", "ip", "209.160.24.63", "avg(response)", 302d)), + ExprValueUtils.tupleValue( + ImmutableMap.of("action", "GET", "ip", "112.111.162.4", "avg(response)", 200d)), + ExprValueUtils.tupleValue( + ImmutableMap.of("action", "POST", "ip", "74.125.19.106", "avg(response)", 350d)) + )); + } + + @Test + public void sum_with_one_groups() { + PhysicalPlan plan = new AggregationOperator(new TestScan(), + Collections.singletonList(dsl.sum(DSL.ref("response", INTEGER))), + Collections.singletonList(DSL.ref("action", STRING))); + List result = execute(plan); + assertEquals(2, result.size()); + assertThat(result, containsInAnyOrder( + ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET", "sum(response)", 804)), + ExprValueUtils.tupleValue(ImmutableMap.of("action", "POST", "sum(response)", 700)) + )); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/DedupeOperatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/DedupeOperatorTest.java new file mode 100644 index 0000000000..29e2308e71 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/DedupeOperatorTest.java @@ -0,0 +1,271 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.tupleValue; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlanDSL.dedupe; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.DedupeOperator.Deduper; +import com.google.common.collect.ImmutableMap; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class DedupeOperatorTest extends PhysicalPlanTestBase { + @Mock + private PhysicalPlan inputPlan; + + /** + * construct the map which contain null value, because {@link ImmutableMap} doesn't support null + * value. + */ + private static final Map NULL_MAP = + new HashMap() { + { + put("region", null); + put("action", "GET"); + } + }; + + @Test + public void dedupe_one_field() { + when(inputPlan.hasNext()).thenReturn(true, true, true, false); + when(inputPlan.next()) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "GET", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "POST", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "PUT", "response", 200))); + + assertThat( + execute(dedupe(inputPlan, DSL.ref("region", STRING))), + contains( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "GET", "response", 200)))); + } + + @Test + public void dedupe_one_field_no_duplication() { + when(inputPlan.hasNext()).thenReturn(true, true, true, false); + when(inputPlan.next()) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "GET", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "POST", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "PUT", "response", 200))); + PhysicalPlan plan = dedupe(inputPlan, DSL.ref("action", STRING)); + + assertThat( + execute(plan), + contains( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "GET", "response", 200)), + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "POST", "response", 200)), + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "PUT", "response", 200)))); + } + + @Test + public void dedupe_one_field_allow_2_duplication() { + when(inputPlan.hasNext()).thenReturn(true, true, true, false); + when(inputPlan.next()) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "GET", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "POST", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "PUT", "response", 200))); + + assertThat( + execute(dedupe(inputPlan, 2, false, false, DSL.ref("region", STRING))), + contains( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "GET", "response", 200)), + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "POST", "response", 200)))); + } + + @Test + public void dedupe_one_field_in_consecutive_mode() { + when(inputPlan.hasNext()).thenReturn(true, true, true, true, false); + when(inputPlan.next()) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "GET", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "POST", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-west-2", "action", "POST", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "PUT", "response", 200))); + + assertThat( + execute(dedupe(inputPlan, 1, false, true, DSL.ref("region", STRING))), + contains( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "GET", "response", 200)), + tupleValue(ImmutableMap.of("region", "us-west-2", "action", "POST", "response", 200)), + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "PUT", "response", 200)))); + } + + @Test + public void dedupe_one_field_in_consecutive_mode_all_consecutive() { + when(inputPlan.hasNext()).thenReturn(true, true, true, false); + when(inputPlan.next()) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "GET", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "POST", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "PUT", "response", 200))); + + assertThat( + execute(dedupe(inputPlan, 1, false, true, DSL.ref("region", STRING))), + contains( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "GET", "response", 200)))); + } + + @Test + public void dedupe_one_field_in_consecutive_mode_allow_2_duplication() { + when(inputPlan.hasNext()).thenReturn(true, true, true, true, false); + when(inputPlan.next()) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "GET", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "POST", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-west-2", "action", "POST", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "PUT", "response", 200))); + + assertThat( + execute(dedupe(inputPlan, 2, false, true, DSL.ref("region", STRING))), + contains( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "GET", "response", 200)), + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "POST", "response", 200)), + tupleValue(ImmutableMap.of("region", "us-west-2", "action", "POST", "response", 200)), + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "PUT", "response", 200)))); + } + + @Test + public void dedupe_two_field() { + when(inputPlan.hasNext()).thenReturn(true, true, true, false); + when(inputPlan.next()) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "GET", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "POST", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "GET", "response", 200))); + + assertThat( + execute(dedupe(inputPlan, DSL.ref("region", STRING), DSL.ref("action", STRING))), + contains( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "GET", "response", 200)), + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "POST", "response", 200)))); + } + + @Test + public void dedupe_one_field_with_missing_value() { + when(inputPlan.hasNext()).thenReturn(true, true, true, false); + when(inputPlan.next()) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "GET", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "POST", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of("action", "POST", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "GET", "response", 200))); + + assertThat( + execute(dedupe(inputPlan, DSL.ref("region", STRING))), + contains( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "GET", "response", 200)))); + } + + @Test + public void dedupe_one_field_with_missing_value_keep_empty() { + when(inputPlan.hasNext()).thenReturn(true, true, true, false); + when(inputPlan.next()) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "GET", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "POST", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of("action", "POST", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "GET", "response", 200))); + + assertThat( + execute(dedupe(inputPlan, 1, true, false, DSL.ref("region", STRING))), + contains( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "GET", "response", 200)), + tupleValue(ImmutableMap.of("action", "POST", "response", 200)))); + } + + @Test + public void dedupe_one_field_with_null_value() { + when(inputPlan.hasNext()).thenReturn(true, true, true, false); + when(inputPlan.next()) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "GET", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "POST", "response", 200))) + .thenReturn( + tupleValue(NULL_MAP)) + .thenReturn( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "GET", "response", 200))); + + assertThat( + execute(dedupe(inputPlan, DSL.ref("region", STRING))), + contains( + tupleValue(ImmutableMap.of("region", "us-east-1", "action", "GET", "response", 200)))); + } + + @Test + public void historical_deduper() { + Deduper deduper = Deduper.historicalDeduper(); + + // first time seen 1 + assertEquals(1, deduper.seenTimes(1)); + // second time seen 1 + assertEquals(2, deduper.seenTimes(1)); + // first time seen 2 + assertEquals(1, deduper.seenTimes(2)); + // third time seen 1 + assertEquals(3, deduper.seenTimes(1)); + } + + @Test + public void consecutive_deduper() { + Deduper deduper = Deduper.consecutiveDeduper(); + + // first time seen 1 + assertEquals(1, deduper.seenTimes(1)); + // consecutive second time seen 1 + assertEquals(2, deduper.seenTimes(1)); + // first time seen 2 + assertEquals(1, deduper.seenTimes(2)); + // first time seen 1 + assertEquals(1, deduper.seenTimes(1)); + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/EvalOperatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/EvalOperatorTest.java new file mode 100644 index 0000000000..1a9e35bbb5 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/EvalOperatorTest.java @@ -0,0 +1,144 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlanDSL.eval; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.Matchers.iterableWithSize; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.google.common.collect.ImmutableMap; +import java.util.List; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class EvalOperatorTest extends PhysicalPlanTestBase { + @Mock + private PhysicalPlan inputPlan; + + @Test + public void create_new_field_that_contain_the_result_of_a_calculation() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()) + .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("distance", 100, "time", 10))); + + PhysicalPlan plan = + eval( + inputPlan, + ImmutablePair.of( + DSL.ref("velocity", DOUBLE), + dsl.divide(DSL.ref("distance", INTEGER), DSL.ref("time", INTEGER)))); + assertThat( + execute(plan), + allOf( + iterableWithSize(1), + hasItems( + ExprValueUtils.tupleValue( + ImmutableMap.of("distance", 100, "time", 10, "velocity", 10))))); + } + + @Test + public void create_multiple_field_using_field_defined_in_input_tuple() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()) + .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("distance", 100, "time", 10))); + + PhysicalPlan plan = + eval( + inputPlan, + ImmutablePair.of( + DSL.ref("velocity", DOUBLE), dsl.divide(DSL.ref("distance", INTEGER), DSL.ref( + "time", INTEGER))), + ImmutablePair.of( + DSL.ref("doubleDistance", INTEGER), + dsl.multiply(DSL.ref("distance", INTEGER), DSL.literal(2)))); + assertThat( + execute(plan), + allOf( + iterableWithSize(1), + hasItems( + ExprValueUtils.tupleValue( + ImmutableMap.of( + "distance", 100, "time", 10, "velocity", 10, "doubleDistance", 200))))); + } + + @Test + public void create_multiple_filed_using_field_defined_in_eval_operator() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()) + .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("distance", 100, "time", 10))); + + PhysicalPlan plan = + eval( + inputPlan, + ImmutablePair.of( + DSL.ref("velocity", INTEGER), dsl.divide(DSL.ref("distance", INTEGER), DSL.ref( + "time", INTEGER))), + ImmutablePair.of( + DSL.ref("doubleVelocity", INTEGER), + dsl.multiply(DSL.ref("velocity", INTEGER), DSL.literal(2)))); + assertThat( + execute(plan), + allOf( + iterableWithSize(1), + hasItems( + ExprValueUtils.tupleValue( + ImmutableMap.of( + "distance", 100, "time", 10, "velocity", 10, "doubleVelocity", 20))))); + } + + @Test + public void replace_existing_field() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()) + .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("distance", 100, "time", 10))); + + PhysicalPlan plan = + eval( + inputPlan, + ImmutablePair.of( + DSL.ref("distance", INTEGER), + dsl.multiply(DSL.ref("distance", INTEGER), DSL.literal(2)))); + assertThat( + execute(plan), + allOf( + iterableWithSize(1), + hasItems(ExprValueUtils.tupleValue(ImmutableMap.of("distance", 200, "time", 10))))); + } + + @Test + public void do_nothing_with_none_tuple_value() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()).thenReturn(ExprValueUtils.integerValue(1)); + PhysicalPlan plan = eval(inputPlan, ImmutablePair.of(DSL.ref("response", INTEGER), + DSL.ref("referer", STRING))); + List result = execute(plan); + + assertThat(result, allOf(iterableWithSize(1), hasItems(ExprValueUtils.integerValue(1)))); + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/FilterOperatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/FilterOperatorTest.java new file mode 100644 index 0000000000..b95e039917 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/FilterOperatorTest.java @@ -0,0 +1,43 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.google.common.collect.ImmutableMap; +import java.util.List; +import org.junit.jupiter.api.Test; + +class FilterOperatorTest extends PhysicalPlanTestBase { + + @Test + public void filterTest() { + FilterOperator plan = new FilterOperator(new TestScan(), + dsl.equal(DSL.ref("response", INTEGER), DSL.literal(404))); + List result = execute(plan); + assertEquals(1, result.size()); + assertThat(result, containsInAnyOrder(ExprValueUtils + .tupleValue(ImmutableMap + .of("ip", "209.160.24.63", "action", "GET", "response", 404, "referer", + "www.amazon.com")))); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java new file mode 100644 index 0000000000..4a6de485cc --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java @@ -0,0 +1,163 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort.SortOption; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.util.Collections; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +/** + * Todo, testing purpose, delete later. + */ +@ExtendWith(MockitoExtension.class) +class PhysicalPlanNodeVisitorTest extends PhysicalPlanTestBase { + @Mock + PhysicalPlan plan; + @Mock + ReferenceExpression ref; + + @Test + public void print_physical_plan() { + PhysicalPlan plan = + PhysicalPlanDSL.remove( + PhysicalPlanDSL.project( + PhysicalPlanDSL.rename( + PhysicalPlanDSL.agg( + PhysicalPlanDSL.filter( + new TestScan(), + dsl.equal(DSL.ref("response", INTEGER), DSL.literal(10))), + ImmutableList.of(dsl.avg(DSL.ref("response", INTEGER))), + ImmutableList.of()), + ImmutableMap.of(DSL.ref("ivalue", INTEGER), DSL.ref("avg(response)", DOUBLE))), + ref), + ref); + + PhysicalPlanPrinter printer = new PhysicalPlanPrinter(); + assertEquals( + "Remove->\n" + + "\tProject->\n" + + "\t\tRename->\n" + + "\t\t\tAggregation->\n" + + "\t\t\t\tFilter->", + printer.print(plan)); + } + + @Test + public void test_PhysicalPlanVisitor_should_return_null() { + PhysicalPlan filter = + PhysicalPlanDSL.filter( + new TestScan(), dsl.equal(DSL.ref("response", INTEGER), DSL.literal(10))); + assertNull(filter.accept(new PhysicalPlanNodeVisitor() { + }, null)); + + PhysicalPlan aggregation = + PhysicalPlanDSL.agg( + filter, ImmutableList.of(dsl.avg(DSL.ref("response", INTEGER))), ImmutableList.of()); + assertNull(aggregation.accept(new PhysicalPlanNodeVisitor() { + }, null)); + + PhysicalPlan rename = + PhysicalPlanDSL.rename( + aggregation, ImmutableMap.of(DSL.ref("ivalue", INTEGER), DSL.ref("avg(response)", + DOUBLE))); + assertNull(rename.accept(new PhysicalPlanNodeVisitor() { + }, null)); + + PhysicalPlan project = PhysicalPlanDSL.project(plan, ref); + assertNull(project.accept(new PhysicalPlanNodeVisitor() { + }, null)); + + PhysicalPlan remove = PhysicalPlanDSL.remove(plan, ref); + assertNull(remove.accept(new PhysicalPlanNodeVisitor() { + }, null)); + + PhysicalPlan eval = PhysicalPlanDSL.eval(plan, Pair.of(ref, ref)); + assertNull(eval.accept(new PhysicalPlanNodeVisitor() { + }, null)); + + PhysicalPlan sort = PhysicalPlanDSL.sort(plan, 100, Pair.of(SortOption.PPL_ASC, ref)); + assertNull(sort.accept(new PhysicalPlanNodeVisitor() { + }, null)); + + PhysicalPlan dedupe = PhysicalPlanDSL.dedupe(plan, ref); + assertNull(dedupe.accept(new PhysicalPlanNodeVisitor() { + }, null)); + + PhysicalPlan values = PhysicalPlanDSL.values(Collections.emptyList()); + assertNull(values.accept(new PhysicalPlanNodeVisitor() { + }, null)); + } + + public static class PhysicalPlanPrinter extends PhysicalPlanNodeVisitor { + + public String print(PhysicalPlan node) { + return node.accept(this, 0); + } + + @Override + public String visitFilter(FilterOperator node, Integer tabs) { + return name(node, "Filter->", tabs); + } + + @Override + public String visitAggregation(AggregationOperator node, Integer tabs) { + return name(node, "Aggregation->", tabs); + } + + @Override + public String visitRename(RenameOperator node, Integer tabs) { + return name(node, "Rename->", tabs); + } + + @Override + public String visitProject(ProjectOperator node, Integer tabs) { + return name(node, "Project->", tabs); + } + + @Override + public String visitRemove(RemoveOperator node, Integer tabs) { + return name(node, "Remove->", tabs); + } + + private String name(PhysicalPlan node, String current, int tabs) { + String child = node.getChild().get(0).accept(this, tabs + 1); + StringBuilder sb = new StringBuilder(); + for (Integer i = 0; i < tabs; i++) { + sb.append("\t"); + } + sb.append(current); + if (!Strings.isNullOrEmpty(child)) { + sb.append("\n"); + sb.append(child); + } + return sb.toString(); + } + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanTestBase.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanTestBase.java new file mode 100644 index 0000000000..9bc4fdc38b --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanTestBase.java @@ -0,0 +1,121 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.config.ExpressionConfig; +import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +@Configuration +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {ExpressionConfig.class}) +public class PhysicalPlanTestBase { + @Autowired + protected DSL dsl; + + private static final List inputs = new ImmutableList.Builder() + .add(ExprValueUtils.tupleValue(ImmutableMap + .of("ip", "209.160.24.63", "action", "GET", "response", 200, "referer", + "www.amazon.com"))) + .add(ExprValueUtils.tupleValue(ImmutableMap + .of("ip", "209.160.24.63", "action", "GET", "response", 404, "referer", + "www.amazon.com"))) + .add(ExprValueUtils.tupleValue(ImmutableMap + .of("ip", "112.111.162.4", "action", "GET", "response", 200, "referer", + "www.amazon.com"))) + .add(ExprValueUtils.tupleValue(ImmutableMap + .of("ip", "74.125.19.106", "action", "POST", "response", 200, "referer", + "www.google.com"))) + .add(ExprValueUtils + .tupleValue(ImmutableMap.of("ip", "74.125.19.106", "action", "POST", "response", 500))) + .build(); + + private static Map typeMapping = + new ImmutableMap.Builder() + .put("ip", ExprCoreType.STRING) + .put("action", ExprCoreType.STRING) + .put("response", ExprCoreType.INTEGER) + .put("referer", ExprCoreType.STRING) + .build(); + + @Bean + protected Environment typeEnv() { + return var -> { + if (var instanceof ReferenceExpression) { + ReferenceExpression refExpr = (ReferenceExpression) var; + if (typeMapping.containsKey(refExpr.getAttr())) { + return typeMapping.get(refExpr.getAttr()); + } + } + throw new ExpressionEvaluationException("type resolved failed"); + }; + } + + protected List execute(PhysicalPlan plan) { + ImmutableList.Builder builder = new ImmutableList.Builder<>(); + plan.open(); + while (plan.hasNext()) { + builder.add(plan.next()); + } + plan.close(); + return builder.build(); + } + + protected static class TestScan extends PhysicalPlan { + private final Iterator iterator; + + public TestScan() { + iterator = inputs.iterator(); + } + + @Override + public R accept(PhysicalPlanNodeVisitor visitor, C context) { + return null; + } + + @Override + public List getChild() { + return ImmutableList.of(); + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public ExprValue next() { + return iterator.next(); + } + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperatorTest.java new file mode 100644 index 0000000000..09d97b19d8 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperatorTest.java @@ -0,0 +1,91 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlanDSL.project; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.Matchers.iterableWithSize; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.google.common.collect.ImmutableMap; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class ProjectOperatorTest extends PhysicalPlanTestBase { + + @Mock + private PhysicalPlan inputPlan; + + @Test + public void project_one_field() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()) + .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET", "response", 200))); + PhysicalPlan plan = project(inputPlan, DSL.ref("action", STRING)); + List result = execute(plan); + + assertThat( + result, + allOf( + iterableWithSize(1), + hasItems(ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET"))))); + } + + @Test + public void project_two_field_follow_the_project_order() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()) + .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET", "response", 200))); + PhysicalPlan plan = project(inputPlan, DSL.ref("response", INTEGER), DSL.ref("action", STRING)); + List result = execute(plan); + + assertThat( + result, + allOf( + iterableWithSize(1), + hasItems( + ExprValueUtils.tupleValue(ImmutableMap.of("response", 200, "action", "GET"))))); + } + + @Test + public void project_ignore_missing_value() { + when(inputPlan.hasNext()).thenReturn(true, true, false); + when(inputPlan.next()) + .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET", "response", 200))) + .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("action", "POST"))); + PhysicalPlan plan = project(inputPlan, DSL.ref("response", INTEGER), DSL.ref("action", STRING)); + List result = execute(plan); + + assertThat( + result, + allOf( + iterableWithSize(2), + hasItems( + ExprValueUtils.tupleValue(ImmutableMap.of("response", 200, "action", "GET")), + ExprValueUtils.tupleValue(ImmutableMap.of("action", "POST"))))); + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RemoveOperatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RemoveOperatorTest.java new file mode 100644 index 0000000000..9d6c9680d3 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RemoveOperatorTest.java @@ -0,0 +1,120 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlanDSL.remove; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.Matchers.iterableWithSize; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.google.common.collect.ImmutableMap; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class RemoveOperatorTest extends PhysicalPlanTestBase { + @Mock + private PhysicalPlan inputPlan; + + @Test + public void remove_one_field() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()) + .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET", "response", 200))); + PhysicalPlan plan = remove(inputPlan, DSL.ref("action", STRING)); + List result = execute(plan); + + assertThat( + result, + allOf( + iterableWithSize(1), + hasItems(ExprValueUtils.tupleValue(ImmutableMap.of("response", 200))))); + } + + @Test + public void remove_one_field_follow_the_input_order() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()) + .thenReturn( + ExprValueUtils.tupleValue( + ImmutableMap.of("action", "GET", "response", 200, "referer", "www.amazon.com"))); + PhysicalPlan plan = remove(inputPlan, DSL.ref("response", INTEGER)); + List result = execute(plan); + + assertThat( + result, + allOf( + iterableWithSize(1), + hasItems( + ExprValueUtils.tupleValue( + ImmutableMap.of("action", "GET", "referer", "www.amazon.com"))))); + } + + @Test + public void remove_two_field() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()) + .thenReturn( + ExprValueUtils.tupleValue( + ImmutableMap.of("action", "GET", "response", 200, "referer", "www.amazon.com"))); + PhysicalPlan plan = remove(inputPlan, DSL.ref("response", INTEGER), DSL.ref("referer", STRING)); + List result = execute(plan); + + assertThat( + result, + allOf( + iterableWithSize(1), + hasItems(ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET"))))); + } + + @Test + public void project_ignore_missing_value() { + when(inputPlan.hasNext()).thenReturn(true, true, false); + when(inputPlan.next()) + .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET", "response", 200))) + .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("action", "POST"))); + PhysicalPlan plan = remove(inputPlan, DSL.ref("response", STRING)); + List result = execute(plan); + + assertThat( + result, + allOf( + iterableWithSize(2), + hasItems( + ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET")), + ExprValueUtils.tupleValue(ImmutableMap.of("action", "POST"))))); + } + + @Test + public void remove_nothing_with_none_tuple_value() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()).thenReturn(ExprValueUtils.integerValue(1)); + PhysicalPlan plan = remove(inputPlan, DSL.ref("response", STRING), DSL.ref("referer", STRING)); + List result = execute(plan); + + assertThat(result, allOf(iterableWithSize(1), hasItems(ExprValueUtils.integerValue(1)))); + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RenameOperatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RenameOperatorTest.java new file mode 100644 index 0000000000..ddccf4bbbf --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RenameOperatorTest.java @@ -0,0 +1,70 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.google.common.collect.ImmutableMap; +import java.util.Collections; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class RenameOperatorTest extends PhysicalPlanTestBase { + @Mock + private PhysicalPlan inputPlan; + + @Test + public void avg_aggregation_rename() { + PhysicalPlan plan = new RenameOperator( + new AggregationOperator(new TestScan(), + Collections.singletonList(dsl.avg(DSL.ref("response", INTEGER))), + Collections.singletonList(DSL.ref("action", STRING))), + ImmutableMap.of(DSL.ref("avg(response)", DOUBLE), DSL.ref("avg", DOUBLE)) + ); + List result = execute(plan); + assertEquals(2, result.size()); + assertThat(result, containsInAnyOrder( + ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET", "avg", 268d)), + ExprValueUtils.tupleValue(ImmutableMap.of("action", "POST", "avg", 350d)) + )); + } + + @Test + public void rename_int_value() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()).thenReturn(ExprValueUtils.integerValue(1)); + PhysicalPlan plan = new RenameOperator( + inputPlan, + ImmutableMap.of(DSL.ref("avg(response)", DOUBLE), DSL.ref("avg", DOUBLE)) + ); + List result = execute(plan); + assertEquals(1, result.size()); + assertThat(result, containsInAnyOrder(ExprValueUtils.integerValue(1))); + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/SortOperatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/SortOperatorTest.java new file mode 100644 index 0000000000..c06ea62c21 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/SortOperatorTest.java @@ -0,0 +1,295 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.tupleValue; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.ref; +import static com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlanDSL.sort; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort.SortOption; +import com.google.common.collect.ImmutableMap; +import java.util.HashMap; +import java.util.Map; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class SortOperatorTest extends PhysicalPlanTestBase { + @Mock + private PhysicalPlan inputPlan; + + /** + * construct the map which contain null value, because {@link ImmutableMap} doesn't support null + * value. + */ + private static final Map NULL_MAP = + new HashMap() { + { + put("size", 399); + put("response", null); + } + }; + + @Test + public void sort_one_field_asc() { + when(inputPlan.hasNext()).thenReturn(true, true, true, false); + when(inputPlan.next()) + .thenReturn(tupleValue(ImmutableMap.of("size", 499, "response", 404))) + .thenReturn(tupleValue(ImmutableMap.of("size", 320, "response", 200))) + .thenReturn(tupleValue(ImmutableMap.of("size", 399, "response", 503))); + + assertThat( + execute(sort(inputPlan, 100, Pair.of(SortOption.PPL_ASC, ref("response", INTEGER)))), + contains( + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 399, "response", 503)))); + } + + @Test + public void sort_one_field_with_duplication() { + when(inputPlan.hasNext()).thenReturn(true, true, true, false); + when(inputPlan.next()) + .thenReturn(tupleValue(ImmutableMap.of("size", 499, "response", 404))) + .thenReturn(tupleValue(ImmutableMap.of("size", 320, "response", 404))) + .thenReturn(tupleValue(ImmutableMap.of("size", 399, "response", 503))); + + assertThat( + execute(sort(inputPlan, 100, Pair.of(SortOption.PPL_ASC, ref("response", INTEGER)))), + contains( + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 320, "response", 404)), + tupleValue(ImmutableMap.of("size", 399, "response", 503)))); + } + + @Test + public void sort_one_field_asc_with_null_value() { + when(inputPlan.hasNext()).thenReturn(true, true, true, true, false); + when(inputPlan.next()) + .thenReturn(tupleValue(ImmutableMap.of("size", 499, "response", 404))) + .thenReturn(tupleValue(ImmutableMap.of("size", 320, "response", 200))) + .thenReturn(tupleValue(ImmutableMap.of("size", 399, "response", 503))) + .thenReturn(tupleValue(NULL_MAP)); + + assertThat( + execute(sort(inputPlan, 100, Pair.of(SortOption.PPL_ASC, ref("response", INTEGER)))), + contains( + tupleValue(NULL_MAP), + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 399, "response", 503)))); + } + + @Test + public void sort_one_field_asc_with_missing_value() { + when(inputPlan.hasNext()).thenReturn(true, true, true, true, false); + when(inputPlan.next()) + .thenReturn(tupleValue(ImmutableMap.of("size", 499, "response", 404))) + .thenReturn(tupleValue(ImmutableMap.of("size", 320, "response", 200))) + .thenReturn(tupleValue(ImmutableMap.of("size", 399, "response", 503))) + .thenReturn(tupleValue(ImmutableMap.of("size", 399))); + + assertThat( + execute(sort(inputPlan, 100, Pair.of(SortOption.PPL_ASC, ref("response", INTEGER)))), + contains( + tupleValue(ImmutableMap.of("size", 399)), + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 399, "response", 503)))); + } + + @Test + public void sort_one_field_desc() { + when(inputPlan.hasNext()).thenReturn(true, true, true, false); + when(inputPlan.next()) + .thenReturn(tupleValue(ImmutableMap.of("size", 499, "response", 404))) + .thenReturn(tupleValue(ImmutableMap.of("size", 320, "response", 200))) + .thenReturn(tupleValue(ImmutableMap.of("size", 399, "response", 503))); + + assertThat( + execute(sort(inputPlan, 100, Pair.of(SortOption.PPL_DESC, ref("response", INTEGER)))), + contains( + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 320, "response", 200)))); + } + + @Test + public void sort_one_field_desc_with_null_value() { + when(inputPlan.hasNext()).thenReturn(true, true, true, true, false); + when(inputPlan.next()) + .thenReturn(tupleValue(NULL_MAP)) + .thenReturn(tupleValue(ImmutableMap.of("size", 499, "response", 404))) + .thenReturn(tupleValue(ImmutableMap.of("size", 320, "response", 200))) + .thenReturn(tupleValue(ImmutableMap.of("size", 399, "response", 503))); + + assertThat( + execute(sort(inputPlan, 100, Pair.of(SortOption.PPL_DESC, ref("response", INTEGER)))), + contains( + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(NULL_MAP))); + } + + @Test + public void sort_one_field_with_duplicate_value() { + when(inputPlan.hasNext()).thenReturn(true, true, true, true, false); + when(inputPlan.next()) + .thenReturn(tupleValue(ImmutableMap.of("size", 499, "response", 404))) + .thenReturn(tupleValue(ImmutableMap.of("size", 320, "response", 200))) + .thenReturn(tupleValue(ImmutableMap.of("size", 499, "response", 404))) + .thenReturn(tupleValue(ImmutableMap.of("size", 399, "response", 503))); + + assertThat( + execute(sort(inputPlan, 100, Pair.of(SortOption.PPL_ASC, ref("response", INTEGER)))), + contains( + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 399, "response", 503)))); + } + + @Test + public void sort_two_fields_both_asc() { + when(inputPlan.hasNext()).thenReturn(true, true, true, true, true, false); + when(inputPlan.next()) + .thenReturn(tupleValue(ImmutableMap.of("size", 499, "response", 404))) + .thenReturn(tupleValue(ImmutableMap.of("size", 320, "response", 200))) + .thenReturn(tupleValue(ImmutableMap.of("size", 399, "response", 200))) + .thenReturn(tupleValue(ImmutableMap.of("size", 399, "response", 503))) + .thenReturn(tupleValue(NULL_MAP)); + + assertThat( + execute( + sort( + inputPlan, + 100, + Pair.of(SortOption.PPL_ASC, ref("size", INTEGER)), + Pair.of(SortOption.PPL_ASC, ref("response", INTEGER)))), + contains( + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(NULL_MAP), + tupleValue(ImmutableMap.of("size", 399, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(ImmutableMap.of("size", 499, "response", 404)))); + } + + @Test + public void sort_two_fields_both_desc() { + when(inputPlan.hasNext()).thenReturn(true, true, true, true, true, false); + when(inputPlan.next()) + .thenReturn(tupleValue(ImmutableMap.of("size", 499, "response", 404))) + .thenReturn(tupleValue(ImmutableMap.of("size", 320, "response", 200))) + .thenReturn(tupleValue(ImmutableMap.of("size", 399, "response", 200))) + .thenReturn(tupleValue(ImmutableMap.of("size", 399, "response", 503))) + .thenReturn(tupleValue(NULL_MAP)); + + assertThat( + execute( + sort( + inputPlan, + 100, + Pair.of(SortOption.PPL_DESC, ref("size", INTEGER)), + Pair.of(SortOption.PPL_DESC, ref("response", INTEGER)))), + contains( + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(ImmutableMap.of("size", 399, "response", 200)), + tupleValue(NULL_MAP), + tupleValue(ImmutableMap.of("size", 320, "response", 200)))); + } + + @Test + public void sort_two_fields_asc_and_desc() { + when(inputPlan.hasNext()).thenReturn(true, true, true, true, true, false); + when(inputPlan.next()) + .thenReturn(tupleValue(ImmutableMap.of("size", 499, "response", 404))) + .thenReturn(tupleValue(ImmutableMap.of("size", 320, "response", 200))) + .thenReturn(tupleValue(ImmutableMap.of("size", 399, "response", 200))) + .thenReturn(tupleValue(ImmutableMap.of("size", 399, "response", 503))) + .thenReturn(tupleValue(NULL_MAP)); + + assertThat( + execute( + sort( + inputPlan, + 100, + Pair.of(SortOption.PPL_ASC, ref("size", INTEGER)), + Pair.of(SortOption.PPL_DESC, ref("response", INTEGER)))), + contains( + tupleValue(ImmutableMap.of("size", 320, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(ImmutableMap.of("size", 399, "response", 200)), + tupleValue(NULL_MAP), + tupleValue(ImmutableMap.of("size", 499, "response", 404)))); + } + + @Test + public void sort_two_fields_desc_and_asc() { + when(inputPlan.hasNext()).thenReturn(true, true, true, true, true, false); + when(inputPlan.next()) + .thenReturn(tupleValue(ImmutableMap.of("size", 499, "response", 404))) + .thenReturn(tupleValue(ImmutableMap.of("size", 320, "response", 200))) + .thenReturn(tupleValue(ImmutableMap.of("size", 399, "response", 200))) + .thenReturn(tupleValue(ImmutableMap.of("size", 399, "response", 503))) + .thenReturn(tupleValue(NULL_MAP)); + + assertThat( + execute( + sort( + inputPlan, + 100, + Pair.of(SortOption.PPL_DESC, ref("size", INTEGER)), + Pair.of(SortOption.PPL_ASC, ref("response", INTEGER)))), + contains( + tupleValue(ImmutableMap.of("size", 499, "response", 404)), + tupleValue(NULL_MAP), + tupleValue(ImmutableMap.of("size", 399, "response", 200)), + tupleValue(ImmutableMap.of("size", 399, "response", 503)), + tupleValue(ImmutableMap.of("size", 320, "response", 200)))); + } + + @Test + public void sort_one_field_asc_with_count() { + when(inputPlan.hasNext()).thenReturn(true, true, true, false); + when(inputPlan.next()) + .thenReturn(tupleValue(ImmutableMap.of("size", 499, "response", 404))) + .thenReturn(tupleValue(ImmutableMap.of("size", 320, "response", 200))) + .thenReturn(tupleValue(ImmutableMap.of("size", 399, "response", 503))); + + assertThat( + execute(sort(inputPlan, 1, Pair.of(SortOption.PPL_ASC, ref("response", INTEGER)))), + contains(tupleValue(ImmutableMap.of("size", 320, "response", 200)))); + } + + @Test + public void sort_one_field_without_input() { + when(inputPlan.hasNext()).thenReturn(false); + + assertEquals( + 0, + execute(sort(inputPlan, 1, Pair.of(SortOption.PPL_ASC, ref("response", INTEGER)))).size()); + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ValuesOperatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ValuesOperatorTest.java new file mode 100644 index 0000000000..58f5acef76 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ValuesOperatorTest.java @@ -0,0 +1,59 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.collectionValue; +import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.literal; +import static com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlanDSL.values; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.is; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.google.common.collect.ImmutableList; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.Test; + +class ValuesOperatorTest { + + @Test + public void shouldHaveNoChild() { + ValuesOperator values = values(ImmutableList.of(literal(1))); + assertThat( + values.getChild(), + is(empty()) + ); + } + + @Test + public void iterateSingleRow() { + ValuesOperator values = values(ImmutableList.of(literal(1), literal("abc"))); + List results = new ArrayList<>(); + while (values.hasNext()) { + results.add(values.next()); + } + + assertThat( + results, + contains(collectionValue(Arrays.asList(1, "abc"))) + ); + } + +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/storage/TableScanOperatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/storage/TableScanOperatorTest.java new file mode 100644 index 0000000000..25869deee9 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/storage/TableScanOperatorTest.java @@ -0,0 +1,62 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.storage; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlanNodeVisitor; +import org.junit.jupiter.api.Test; + +class TableScanOperatorTest { + + private final TableScanOperator tableScan = new TableScanOperator() { + @Override + public boolean hasNext() { + return false; + } + + @Override + public ExprValue next() { + return null; + } + }; + + @Test + public void accept() { + Boolean isVisited = tableScan.accept(new PhysicalPlanNodeVisitor() { + @Override + protected Boolean visitNode(PhysicalPlan node, Object context) { + return (node instanceof TableScanOperator); + } + + @Override + public Boolean visitTableScan(TableScanOperator node, Object context) { + return super.visitTableScan(node, context); + } + }, null); + + assertTrue(isVisited); + } + + @Test + public void getChild() { + assertTrue(tableScan.getChild().isEmpty()); + } + +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/storage/bindingtuple/BindingTupleTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/storage/bindingtuple/BindingTupleTest.java new file mode 100644 index 0000000000..2def549f68 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/storage/bindingtuple/BindingTupleTest.java @@ -0,0 +1,59 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Test; + +class BindingTupleTest { + @Test + public void resolve_ref_expression() { + BindingTuple bindingTuple = + ExprValueUtils.tupleValue(ImmutableMap.of("ip", "209.160.24.63")).bindingTuples(); + assertEquals(ExprValueUtils.stringValue("209.160.24.63"), + bindingTuple.resolve(DSL.ref("ip", STRING))); + } + + @Test + public void resolve_missing_expression() { + BindingTuple bindingTuple = + ExprValueUtils.tupleValue(ImmutableMap.of("ip", "209.160.24.63")).bindingTuples(); + assertEquals(ExprValueUtils.LITERAL_MISSING, + bindingTuple.resolve(DSL.ref("ip_missing", STRING))); + } + + @Test + public void resolve_from_empty_tuple() { + assertEquals(ExprValueUtils.LITERAL_MISSING, + BindingTuple.EMPTY.resolve(DSL.ref("ip_missing", STRING))); + } + + @Test + public void resolve_literal_expression_throw_exception() { + BindingTuple bindingTuple = + ExprValueUtils.tupleValue(ImmutableMap.of("ip", "209.160.24.63")).bindingTuples(); + ExpressionEvaluationException exception = assertThrows(ExpressionEvaluationException.class, + () -> bindingTuple.resolve(DSL.literal(1))); + assertEquals("can resolve expression: 1", exception.getMessage()); + } +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/utils/ComparisonUtil.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/utils/ComparisonUtil.java new file mode 100644 index 0000000000..5c8155e75c --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/utils/ComparisonUtil.java @@ -0,0 +1,59 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.utils; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getDoubleValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getFloatValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getIntegerValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getLongValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getStringValue; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDoubleValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprFloatValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprIntegerValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprLongValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprStringValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; + +public class ComparisonUtil { + /** + * Util to compare the object (integer, long, float, double, string) values. + * ExprValue A + */ + public static int compare(ExprValue v1, ExprValue v2) { + if (v1.isMissing() || v2.isMissing()) { + throw new ExpressionEvaluationException("invalid to call compare operation on missing value"); + } else if (v1.isNull() || v2.isNull()) { + throw new ExpressionEvaluationException("invalid to call compare operation on null value"); + } + + if (v1 instanceof ExprIntegerValue) { + return getIntegerValue(v1).compareTo(getIntegerValue(v2)); + } else if (v1 instanceof ExprLongValue) { + return getLongValue(v1).compareTo(getLongValue(v2)); + } else if (v1 instanceof ExprFloatValue) { + return getFloatValue(v1).compareTo(getFloatValue(v2)); + } else if (v1 instanceof ExprDoubleValue) { + return getDoubleValue(v1).compareTo(getDoubleValue(v2)); + } else if (v1 instanceof ExprStringValue) { + return getStringValue(v1).compareTo(getStringValue(v2)); + } else { + throw new ExpressionEvaluationException( + String.format("%s instances are not comparable", v1.getClass().getSimpleName())); + } + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/utils/MatcherUtils.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/utils/MatcherUtils.java new file mode 100644 index 0000000000..ac3c1d6db2 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/utils/MatcherUtils.java @@ -0,0 +1,60 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.utils; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import org.hamcrest.Description; +import org.hamcrest.TypeSafeMatcher; + +/** + * Matcher Utils. + */ +public class MatcherUtils { + /** + * Check {@link ExprValue} type equal to {@link ExprCoreType}. + */ + public static TypeSafeMatcher hasType(ExprCoreType type) { + return new TypeSafeMatcher() { + @Override + public void describeTo(Description description) { + description.appendText(type.toString()); + } + + @Override + protected boolean matchesSafely(ExprValue value) { + return type == value.type(); + } + }; + } + + /** + * Check {@link ExprValue} value equal to {@link Object}. + */ + public static TypeSafeMatcher hasValue(Object object) { + return new TypeSafeMatcher() { + @Override + public void describeTo(Description description) { + description.appendText(object.toString()); + } + + @Override + protected boolean matchesSafely(ExprValue value) { + return object.equals(value.value()); + } + }; + } +} diff --git a/docs/category.json b/docs/category.json new file mode 100644 index 0000000000..d889ba4a8d --- /dev/null +++ b/docs/category.json @@ -0,0 +1,20 @@ +{ + "bash": [ + "experiment/ppl/interfaces/endpoint.rst", + "experiment/ppl/interfaces/protocol.rst", + "experiment/ppl/admin/settings.rst" + ], + "ppl_cli": [ + "experiment/ppl/cmd/dedup.rst", + "experiment/ppl/cmd/eval.rst", + "experiment/ppl/cmd/fields.rst", + "experiment/ppl/cmd/rename.rst", + "experiment/ppl/cmd/search.rst", + "experiment/ppl/cmd/sort.rst", + "experiment/ppl/cmd/stats.rst", + "experiment/ppl/cmd/where.rst" + ], + "sql_cli": [ + "user/dql/expressions.rst" + ] +} \ No newline at end of file diff --git a/docs/dev/Doctest.md b/docs/dev/Doctest.md new file mode 100644 index 0000000000..18750834fe --- /dev/null +++ b/docs/dev/Doctest.md @@ -0,0 +1,273 @@ +# Doctest + +# 1. Overview + +## 1.1 What is doctest? + +Doctest is a way to test code by checking the correctness of embedded interactive examples in documentation +[example explaining doctest module](https://docs.python.org/3/library/doctest.html#simple-usage-checking-examples-in-a-text-file) + +`example.txt` + +``` +This is an example text file in reStructuredText format. First import +``factorial`` from the ``example`` module: + + **>>> from example import factorial** + +Now use it: + + **>>> factorial(6) + 120** +``` + + +Running `doctest.testfile("example.txt")` then finds the error + +``` +File "./example.txt", line 14, in example.txt +Failed example: + factorial(6) +Expected: + 120 +Got: + 720 +``` + +## 1.2 Why use doctest? + +Doctest can make the code snippet in docs executable, and check its response as well. + +As mentioned in the python doctest [introduction](https://docs.python.org/3/library/doctest.html#), doctest has grown to have three primary uses: + +1. Checking examples in docstrings. +2. Regression testing. +3. Executable documentation / literate testing. + +## 1.3 How does it affect unit/integration testing suite? + +Both doctest and unit/integration test are valuable. Use doctest for cases where the test is giving an example of usage that is actually useful as documentation. Generally, don't make these tests comprehensive, aiming solely for informative. Use doctest in reverse: **not to test the code is correct based on doctest, but to check that documentation is correct based on the code.** + +For actually testing the code, the goal is to thoroughly test every case, rather than illustrate what it does by example. Doctests aren't meant to be a comprehensive testing solution - they're meant to ensure that simple interactive-prompt style examples in your documentation (including docstrings) don't get out of date. + +## 1.4 How to use doctest? +### 1.4.2 How to run existing doctest? +Doctest runs with project build by `./gradlew build`. You can also only run doctest by `./gradlew doctest` + +Make sure you don't have any Elasticsearch instance running at `http://localhost:9200` + +### 1.4.2 How to write documentation with doctest? +1. If you want to add a new doc, you can add it to `docs` folder, under correct sub-folder, in `.rst` format. +> **Attention**: For code examples in documentation, a Mixing usage of `cli` and `bash` in one doc is not supported yet. +2. Add your new doc file path to `docs/category.json` by its category +3. Run doctest `./gradlew doctest` to see if your tests can pass + +Currently, there is a `sample` folder under `docs` module to help you get started. + +## 1.5 Future Plan + +Current SQL Documentation will need reconstruction in the future. Ideally, both SQL and PPL doctest will integrate/migrate to a separate Doctest module in the new architecture. + +# 2. Design + +## 2.1 Workflow + +![doctest-workflow](img/doctest-workflow.png) + + + +## 2.2 ES Test Instance + +### 2.2.1 Testing framework + +We have two options here + +1. Use ES integration testing framework, same as SQL Plugin integration test. +2. Spin up ES instance with SQL Plugin installed without gradle build the package, simply `./gradlew run` + + +The reason we are not using ES test framework, is due to the difficulty of integrating Python code to a Java based framework, considering we are using python built-in module `doctest` for implementation + +### 2.2.2 Gradle + +1. Create new module/packdage `doctest` under current `opendistro-sql`, and integrate to gradle management + 1. ![doctest-gradle-project-structure](img/doctest-gradle-project-structure.png) + +2. Set up gradle build script, which enables doctest by `./gradlew doctest` +3. Gradle tasks: + 1. bootstrap + 2. StartES + 1. ` ./gradlew run` + 2. https://github.com/elastic/elasticsearch/blob/master/TESTING.asciidoc#running-elasticsearch-from-a-checkout + 3. doctest + 4. StopES +4. Integrate Doctest to project gradle build, which means `./gradlew build` will also run `doctest` + + + +### **2.2.3 Project Structure** + +`bootstrap.sh` set up virtual environment for python module + +``` +doctest +├── bin +│ └── test-docs +├── bootstrap.sh +├── build.gradle +├── docs +│ └── dql +│ ├── basics.rst +│ └── explain.rst +├── requirements.txt +├── test_data +│ └── accounts.json +└── test_docs.py +``` + + + +## 2.3 Parsers + +> **How are Docstring Examples Recognized?** +In most cases, a copy-and-paste of an interactive console session works fine, but doctest isn’t trying to do an exact emulation of any specific Python shell. + + +Doctest is relying on the console/command line to run code examples in documentation. So we need two parsers here. + + +### 2.3.1 CLI parser + +**Reference: [CrateDB Implementation](https://github.com/crate/crate/blob/master/blackbox/test_docs.py)** + +* To recognize its own cli and bash, CrateDB customize the doctest parser by a 3rd party libary: [zc.customdoctests](https://pypi.org/project/zc.customdoctests/) +* CrateDB doctest uses it’s own cli `crash` to run doctests. https://github.com/crate/crash + * https://github.com/crate/crate/blob/master/docs/general/dql/selects.rst + + +Similar to CarateDB using it’s CLI “crash”, we can make use of our own [SQL-CLI](https://github.com/opendistro-for-elasticsearch/sql-cli) + +To support PPL, we need to add PPL support to SQL-CLI. Since PPL and SQL expose similar http endpoint for query and share similar response format. The update won’t be much of work. + +The code example in a doc using `CLI` should be like this + +``` +odfesql> SELECT firstname, lastname FROM accounts; +fetched rows / total rows = 4/4 ++-------------+------------+ +| firstname | lastname | +|-------------+------------| +| Amber | Duke | +| Hattie | Bond | +| Nanette | Bates | +| Dale | Adams | ++-------------+------------+ +``` + + + +### 2.3.2 bash parser + +1. Use [python subprocess](https://docs.python.org/3/library/subprocess.html) to run `curl` command +2. Need to add additional formatter to better display json response + + +The code example in a doc using `bash` should be like this + +``` +sh$ curl -XPOST "localhost:9200/_opendistro/_ppl/" + -H 'Content-Type: application/json' + -d'{ "query": "search source=kibana_sample_data_flights OriginCountry = "IT" + DestiContry = "US" | fields FlightNum, DestCountry, OriginCountry "}' + + { + { + "FlightNum": "ADGH12", + "OriginCountry": "IT", + "DestCountry": "US" + }, + { + "FlightNum": "IRMW49", + "OriginCountry": "IT", + "DestCountry": "US" + }, + ... + } +``` + + + +## 2.3 Test Data + +Use elasticsearch python library to create connection to ES instance. It can load test data into ES instance, and delete test index after testing. + +**Setup:** `bulk` API +**TearDown:** `delete(index=[""])` + + + +## 2.3 Test Report + +### 2.3.1 Print results + +Use python faulthandler from script to print results + +https://docs.python.org/3/library/faulthandler.html + +``` +> Task :doctest:doctest +/Users/szhongna/Desktop/Projects/sql/doctest/docs/dql/basics.rst +Doctest: basics.rst ... ok +/Users/szhongna/Desktop/Projects/sql/doctest/docs/dql/explain.rst +Doctest: explain.rst ... FAIL + +====================================================================== +FAIL: /Users/szhongna/Desktop/Projects/sql/doctest/docs/dql/explain.rst +Doctest: explain.rst + +---------------------------------------------------------------------- +File "/Users/szhongna/Desktop/Projects/sql/doctest/docs/dql/explain.rst", line 6, in explain.rst +Failed example: + pretty_print(sh("""curl -sS -H 'Content-Type: application/json' \ + -X POST localhost:9200/_opendistro/_sql/_explain \ + -d '{"query" : "SELECT firstname, lastname FROM accounts WHERE age > 20"}' + """).stdout.decode("utf-8")) +Expected: + { + "from": 0, + "size": 200, + "query": { + + ... + + } + } +Got: + { + "from": 0, + "size": 10, + "query": { + + ... + + } + } + + +---------------------------------------------------------------------- +Ran 2 tests in 2.963s + +FAILED (failures=1) + +> Task :doctest:doctest FAILED + +FAILURE: Build failed with an exception. + +``` + + + +### 2.3.2 generate report + +* Python tests can’t be integrated to Jacoco test reporting +* TODO: need to figure out a better solution \ No newline at end of file diff --git a/docs/dev/Testing.md b/docs/dev/Testing.md index 96ae0d3930..260adb1141 100644 --- a/docs/dev/Testing.md +++ b/docs/dev/Testing.md @@ -157,7 +157,7 @@ Use default test set and reference databases by `testType` argument given only. Note that for now test data set argument is not supported because it often requires code changes to map more ES data type to JDBC type as well as convert data. ``` -$ ./gradlew integTestRunner -DtestType=comparison +$ ./gradlew :integ-test:comparisonTest [2020-01-06T11:37:57,437][INFO ][c.a.o.s.c.CorrectnessIT ] [performComparisonTest] Starting comparison test ================================= @@ -242,7 +242,7 @@ $ ./gradlew integTestRunner -DtestType=comparison Specify different test case set by `queries` argument: ``` -$ ./gradlew integTestRunner -DtestType=comparison -Dqueries=sanity_integration_tests.txt +$ ./gradlew :integ-test:comparisonTest -Dqueries=sanity_integration_tests.txt ... Test query set : SQL queries (first 5 in 7): @@ -257,7 +257,7 @@ $ ./gradlew integTestRunner -DtestType=comparison -Dqueries=sanity_integration_t Specify external Elasticsearch cluster by `esHost` argument, otherwise an internal Elasticsearch in workspace is in use by default. ``` -$ ./gradlew integTestRunner -DtestType=comparison -DesHost=localhost:9200 +$ ./gradlew :integ-test:comparisonTest -DesHost=localhost:9200 ================================= Tested Database : localhost:9200 @@ -270,7 +270,7 @@ $ ./gradlew integTestRunner -DtestType=comparison -DesHost=localhost:9200 Specify different databases for comparison. `dbUrl` is for database to be tested. `otherDbUrls` is for other databases whose result set be referenced and compared. ``` -$ ./gradlew integTestRunner -DtestType=comparison -Dqueries=sanity_integration_tests.txt -DdbUrl=jdbc:sqlite::memory: +$ ./gradlew :integ-test:comparisonTest -Dqueries=sanity_integration_tests.txt -DdbUrl=jdbc:sqlite::memory: ================================= Tested Database : jdbc:sqlite::memory: @@ -279,7 +279,7 @@ $ ./gradlew integTestRunner -DtestType=comparison -Dqueries=sanity_integration_t H2 = jdbc:h2:mem:test;DB_CLOSE_DELAY=-1 ... -$ ./gradlew integTestRunner -DtestType=comparison -Dqueries=sanity_integration_tests.txt -DdbUrl=jdbc:sqlite::memory: -DotherDbUrls=Unknown=jdbc:h2:mem:test;DB_CLOSE_DELAY=-1 +$ ./gradlew :integ-test:comparisonTest -Dqueries=sanity_integration_tests.txt -DdbUrl=jdbc:sqlite::memory: -DotherDbUrls=Unknown=jdbc:h2:mem:test;DB_CLOSE_DELAY=-1 ================================= Tested Database : jdbc:sqlite::memory: diff --git a/docs/dev/img/doctest-gradle-project-structure.png b/docs/dev/img/doctest-gradle-project-structure.png new file mode 100644 index 0000000000..3ffe69b2f5 Binary files /dev/null and b/docs/dev/img/doctest-gradle-project-structure.png differ diff --git a/docs/dev/img/doctest-workflow.png b/docs/dev/img/doctest-workflow.png new file mode 100644 index 0000000000..9c7c09204e Binary files /dev/null and b/docs/dev/img/doctest-workflow.png differ diff --git a/docs/experiment/ppl/admin/settings.rst b/docs/experiment/ppl/admin/settings.rst new file mode 100644 index 0000000000..afcf1428de --- /dev/null +++ b/docs/experiment/ppl/admin/settings.rst @@ -0,0 +1,48 @@ +.. highlight:: sh + +============ +PPL Settings +============ + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 1 + + +Introduction +============ + +When Elasticsearch bootstraps, PPL plugin will register a few settings in Elasticsearch cluster settings. Most of the settings are able to change dynamically so you can control the behavior of PPL plugin without need to bounce your cluster. + +opendistro.ppl.query.memory_limit +================================= + +Description +----------- + +You can set heap memory usage limit for PPL query. When query running, it will detected whether the heap memory usage under the limit, if not, it will terminated the current query. The default value is: 85% + +Example +------- + +PPL query:: + + sh$ curl -sS -H 'Content-Type: application/json' \ + ... -X PUT localhost:9200/_cluster/settings \ + ... -d '{"persistent" : {"opendistro.ppl.query.memory_limit" : "80%"}}' + { + "acknowledged": true, + "persistent": { + "opendistro": { + "ppl": { + "query": { + "memory_limit": "80%" + } + } + } + }, + "transient": {} + } + diff --git a/docs/experiment/ppl/cmd/dedup.rst b/docs/experiment/ppl/cmd/dedup.rst new file mode 100644 index 0000000000..b5669c646a --- /dev/null +++ b/docs/experiment/ppl/cmd/dedup.rst @@ -0,0 +1,111 @@ +============= +dedup +============= + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + + +Description +============ +| Using ``dedup`` command to remove identical document defined by field from the search result. + + +Syntax +============ +dedup [int] [keepempty=] [consecutive=] + + +* int: optional. The ``dedup`` command retains multiple events for each combination when you specify . The number for must be greater than 0. If you do not specify a number, only the first occurring event is kept. All other duplicates are removed from the results. **Default:** 1 +* keepempty: optional. if true, keep the document if the any field in the field-list has NULL value or field is MISSING. **Default:** false. +* consecutive: optional. If set to true, removes only events with duplicate combinations of values that are consecutive. **Default:** false. +* field-list: mandatory. The comma-delimited field list. At least one field is required. + + +Example 1: Dedup by one field +============================= + +The example show dedup the document with gender field. + +PPL query:: + + od> source=accounts | dedup gender | fields account_number, gender; + fetched rows / total rows = 2/2 + +------------------+----------+ + | account_number | gender | + |------------------+----------| + | 1 | M | + | 13 | F | + +------------------+----------+ + +Example 2: Keep 2 duplicates documents +====================================== + +The example show dedup the document with gender field keep 2 duplication. + +PPL query:: + + od> source=accounts | dedup 2 gender | fields account_number, gender; + fetched rows / total rows = 3/3 + +------------------+----------+ + | account_number | gender | + |------------------+----------| + | 1 | M | + | 6 | M | + | 13 | F | + +------------------+----------+ + +Example 3: Keep or Ignore the empty field by default +============================================ + +The example show dedup the document by keep null value field. + +PPL query:: + + od> source=accounts | dedup email keepempty=true | fields account_number, email; + fetched rows / total rows = 4/4 + +------------------+-----------------------+ + | account_number | email | + |------------------+-----------------------| + | 1 | amberduke@pyrami.com | + | 6 | hattiebond@netagy.com | + | 13 | null | + | 18 | daleadams@boink.com | + +------------------+-----------------------+ + + +The example show dedup the document by ignore the empty value field. + +PPL query:: + + od> source=accounts | dedup email | fields account_number, email; + fetched rows / total rows = 3/3 + +------------------+-----------------------+ + | account_number | email | + |------------------+-----------------------| + | 1 | amberduke@pyrami.com | + | 6 | hattiebond@netagy.com | + | 18 | daleadams@boink.com | + +------------------+-----------------------+ + + +Example 4: Dedup in consecutive document +========================================= + +The example show dedup the consecutive document. + +PPL query:: + + od> source=accounts | dedup gender consecutive=true | fields account_number, gender; + fetched rows / total rows = 3/3 + +------------------+----------+ + | account_number | gender | + |------------------+----------| + | 1 | M | + | 13 | F | + | 18 | M | + +------------------+----------+ + diff --git a/docs/experiment/ppl/cmd/eval.rst b/docs/experiment/ppl/cmd/eval.rst new file mode 100644 index 0000000000..4ec95ad83e --- /dev/null +++ b/docs/experiment/ppl/cmd/eval.rst @@ -0,0 +1,77 @@ +============= +eval +============= + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + + +Description +============ +| The ``eval`` command evaluate the expression and append the result to the search result. + + +Syntax +============ +eval = ["," = ]... + +* field: mandatory. If the field name not exist, a new field is added. If the field name already exists, it will be overrided. +* expression: mandatory. Any expression support by the system. + +Example 1: Create the new field +=============================== + +The example show to create new field doubleAge for each document. The new doubleAge is the evaluation result of age multiply by 2. + +PPL query:: + + od> source=accounts | eval doubleAge = age * 2 | fields age, doubleAge ; + fetched rows / total rows = 4/4 + +-------+-------------+ + | age | doubleAge | + |-------+-------------| + | 32 | 64 | + | 36 | 72 | + | 28 | 56 | + | 33 | 66 | + +-------+-------------+ + + +Example 2: Override the existing field +====================================== + +The example show to override the exist age field with age plus 1. + +PPL query:: + + od> source=accounts | eval age = age + 1 | fields age ; + fetched rows / total rows = 4/4 + +-------+ + | age | + |-------| + | 33 | + | 37 | + | 29 | + | 34 | + +-------+ + +Example 3: Create the new field with field defined in eval +========================================================== + +The example show to create a new field ddAge with field defined in eval command. The new field ddAge is the evaluation result of doubleAge multiply by 2, the doubleAge is defined in the eval command. + +PPL query:: + + od> source=accounts | eval doubleAge = age * 2, ddAge = doubleAge * 2 | fields age, doubleAge, ddAge ; + fetched rows / total rows = 4/4 + +-------+-------------+---------+ + | age | doubleAge | ddAge | + |-------+-------------+---------| + | 32 | 64 | 128 | + | 36 | 72 | 144 | + | 28 | 56 | 112 | + | 33 | 66 | 132 | + +-------+-------------+---------+ \ No newline at end of file diff --git a/docs/experiment/ppl/cmd/fields.rst b/docs/experiment/ppl/cmd/fields.rst new file mode 100644 index 0000000000..ecdf86cb4b --- /dev/null +++ b/docs/experiment/ppl/cmd/fields.rst @@ -0,0 +1,60 @@ +============= +fields +============= + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + + +Description +============ +| Using ``field`` command to keep or remove fields from the search result. + + +Syntax +============ +field [+|-] + +* index: optional. if the plus (+) is used, only the fields specified in the field list will be keep. if the minus (-) is used, all the fields specified in the field list will be removed. **Default** + +* field list: mandatory. comma-delimited keep or remove fields. + + +Example 1: Select specified fields from result +============================================== + +The example show fetch account_number, firstname and lastname fields from search results. + +PPL query:: + + od> source=accounts | fields account_number, firstname, lastname; + fetched rows / total rows = 4/4 + +------------------+-------------+------------+ + | account_number | firstname | lastname | + |------------------+-------------+------------| + | 1 | Amber | Duke | + | 6 | Hattie | Bond | + | 13 | Nanette | Bates | + | 18 | Dale | Adams | + +------------------+-------------+------------+ + +Example 2: Remove specified fields from result +============================================== + +The example show fetch remove account_number field from search results. + +PPL query:: + + od> source=accounts | fields account_number, firstname, lastname | fields - account_number ; + fetched rows / total rows = 4/4 + +-------------+------------+ + | firstname | lastname | + |-------------+------------| + | Amber | Duke | + | Hattie | Bond | + | Nanette | Bates | + | Dale | Adams | + +-------------+------------+ + diff --git a/docs/experiment/ppl/cmd/rename.rst b/docs/experiment/ppl/cmd/rename.rst new file mode 100644 index 0000000000..e708595d5e --- /dev/null +++ b/docs/experiment/ppl/cmd/rename.rst @@ -0,0 +1,61 @@ +============= +rename +============= + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + + +Description +============ +| Using ``rename`` command to rename one or more fields in the search result. + + +Syntax +============ +rename AS ["," AS ]... + +* source-field: mandatory. The name of the field you want to rename. +* field list: mandatory. The name you want to rename to. + + +Example 1: Rename one field +=========================== + +The example show rename one field. + +PPL query:: + + od> source=accounts | rename account_number as an | fields an; + fetched rows / total rows = 4/4 + +------+ + | an | + |------| + | 1 | + | 6 | + | 13 | + | 18 | + +------+ + + +Example 2: Rename multiple fields +================================= + +The example show rename multiple fields. + +PPL query:: + + od> source=accounts | rename account_number as an, employer as emp | fields an, emp; + fetched rows / total rows = 4/4 + +------+---------+ + | an | emp | + |------+---------| + | 1 | Pyrami | + | 6 | Netagy | + | 13 | Quility | + | 18 | null | + +------+---------+ + diff --git a/docs/experiment/ppl/cmd/search.rst b/docs/experiment/ppl/cmd/search.rst new file mode 100644 index 0000000000..d1a943f24f --- /dev/null +++ b/docs/experiment/ppl/cmd/search.rst @@ -0,0 +1,59 @@ +============= +search +============= + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + + +Description +============ +| Using ``search`` command to retrieve document from the index. ``search`` command could be only used as the first command in the PPL query. + + +Syntax +============ +search source= [boolean-expression] + +* search: search keywords, which could be ignore. +* index: mandatory. search command must specify which index to query from. +* bool-expression: optional. any expression which could be evaluated to boolean value. + + +Example 1: Fetch all the data +============================= + +The example show fetch all the document from accounts index. + +PPL query:: + + od> source=accounts; + fetched rows / total rows = 4/4 + +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+ + | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | + |------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------| + | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | + | 6 | Hattie | 671 Bristol Street | 5686 | M | Dante | Netagy | TN | 36 | hattiebond@netagy.com | Bond | + | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | + | 18 | Dale | 467 Hutchinson Court | 4180 | M | Orick | null | MD | 33 | daleadams@boink.com | Adams | + +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+ + +Example 2: Fetch data with condition +==================================== + +The example show fetch all the document from accounts index with . + +PPL query:: + + od> source=accounts account_number=1 or gender="F"; + fetched rows / total rows = 2/2 + +------------------+-------------+--------------------+-----------+----------+--------+------------+---------+-------+----------------------+------------+ + | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | + |------------------+-------------+--------------------+-----------+----------+--------+------------+---------+-------+----------------------+------------| + | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | + | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | + +------------------+-------------+--------------------+-----------+----------+--------+------------+---------+-------+----------------------+------------+ + diff --git a/docs/experiment/ppl/cmd/sort.rst b/docs/experiment/ppl/cmd/sort.rst new file mode 100644 index 0000000000..522199a732 --- /dev/null +++ b/docs/experiment/ppl/cmd/sort.rst @@ -0,0 +1,98 @@ +============= +sort +============= + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + + +Description +============ +| Using ``sort`` command to sorts all the search result by the specified fields. + + +Syntax +============ +sort [count] <[+|-] sort-field>... + + +* count: optional. The maximum number results to return from the sorted result. **Default:** 10000 +* [+|-]: optional. The plus [+] for ascending order and a minus [-] for descending order. **Default:** ascending order. +* sort-field: mandatory. The field used to sort. + + +Example 1: Sort by one field +============================= + +The example show sort all the document with age field in ascending order. + +PPL query:: + + od> source=accounts | sort age | fields account_number, age; + fetched rows / total rows = 4/4 + +------------------+-------+ + | account_number | age | + |------------------+-------| + | 13 | 28 | + | 1 | 32 | + | 18 | 33 | + | 6 | 36 | + +------------------+-------+ + + +Example 2: Sort by one field in descending order +================================================ + +The example show sort all the document with age field in descending order. + +PPL query:: + + od> source=accounts | sort - age | fields account_number, age; + fetched rows / total rows = 4/4 + +------------------+-------+ + | account_number | age | + |------------------+-------| + | 6 | 36 | + | 18 | 33 | + | 1 | 32 | + | 13 | 28 | + +------------------+-------+ + + +Example 3: Specify the number of sorted documents to return +============================================================ + +The example show sort all the document and return 2 documents. + +PPL query:: + + od> source=accounts | sort 2 age | fields account_number, age; + fetched rows / total rows = 2/2 + +------------------+-------+ + | account_number | age | + |------------------+-------| + | 13 | 28 | + | 1 | 32 | + +------------------+-------+ + +Example 4: Sort by multiple field +============================= + +The example show sort all the document with gender field in ascending order and age field in descending. + +PPL query:: + + od> source=accounts | sort + gender, - age | fields account_number, gender, age; + fetched rows / total rows = 4/4 + +------------------+----------+-------+ + | account_number | gender | age | + |------------------+----------+-------| + | 13 | F | 28 | + | 6 | M | 36 | + | 18 | M | 33 | + | 1 | M | 32 | + +------------------+----------+-------+ + diff --git a/docs/experiment/ppl/cmd/stats.rst b/docs/experiment/ppl/cmd/stats.rst new file mode 100644 index 0000000000..2871951949 --- /dev/null +++ b/docs/experiment/ppl/cmd/stats.rst @@ -0,0 +1,74 @@ +============= +stats +============= + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + + +Description +============ +| Using ``stats`` command to calculate the aggregation from search result. + + +Syntax +============ +stats ... [by-clause]... + + +* aggregation: mandatory. A statistical aggregation function. The argument of aggregation must be field. +* by-clause: optional. The one or more fields to group the results by. **Default**: If no is specified, the stats command returns only one row, which is the aggregation over the entire result set. + + +Example 1: Calculate the average of a field +=========================================== + +The example show calculate the average age of all the accounts. + +PPL query:: + + od> source=accounts | stats avg(age); + fetched rows / total rows = 1/1 + +------------+ + | avg(age) | + |------------| + | 32.25 | + +------------+ + + +Example 2: Calculate the average of a field by group +==================================================== + +The example show calculate the average age of all the accounts group by gender. + +PPL query:: + + od> source=accounts | stats avg(age) by gender; + fetched rows / total rows = 2/2 + +----------+--------------------+ + | gender | avg(age) | + |----------+--------------------| + | M | 33.666666666666664 | + | F | 28 | + +----------+--------------------+ + + +Example 3: Calculate the average and sum of a field by group +============================================================ + +The example show calculate the average age and sum age of all the accounts group by gender. + +PPL query:: + + od> source=accounts | stats avg(age), sum(age) by gender; + fetched rows / total rows = 2/2 + +----------+--------------------+------------+ + | gender | avg(age) | sum(age) | + |----------+--------------------+------------| + | M | 33.666666666666664 | 101 | + | F | 28 | 28 | + +----------+--------------------+------------+ + diff --git a/docs/experiment/ppl/cmd/syntax.rst b/docs/experiment/ppl/cmd/syntax.rst new file mode 100644 index 0000000000..7d4a58b386 --- /dev/null +++ b/docs/experiment/ppl/cmd/syntax.rst @@ -0,0 +1,30 @@ +============= +Syntax +============= + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + +Command Order +============= +The PPL query started with ``search`` command to reference a table search from. All the following command could be in any order. In the following example, ``search`` command refer the accounts index as the source, then using fields and where command to do the further processing. + +.. code-block:: + + search source=accounts + | where age > 18 + | fields firstname, lastname + + +Required arguments +================== +Required arguments are shown in angle brackets < >. + + +Optional arguments +================== +Optional arguments are enclosed in square brackets [ ]. + diff --git a/docs/experiment/ppl/cmd/where.rst b/docs/experiment/ppl/cmd/where.rst new file mode 100644 index 0000000000..340f20e834 --- /dev/null +++ b/docs/experiment/ppl/cmd/where.rst @@ -0,0 +1,38 @@ +============= +where +============= + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + + +Description +============ +| The ``where`` command bool-expression to filter the search result. The ``where`` command only return the result when bool-expression evaluated to true. + + +Syntax +============ +where + +* bool-expression: optional. any expression which could be evaluated to boolean value. + +Example 1: Filter result set with condition +=========================================== + +The example show fetch all the document from accounts index with . + +PPL query:: + + od> source=accounts | where account_number=1 or gender="F"; + fetched rows / total rows = 2/2 + +------------------+-------------+--------------------+-----------+----------+--------+------------+---------+-------+----------------------+------------+ + | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | + |------------------+-------------+--------------------+-----------+----------+--------+------------+---------+-------+----------------------+------------| + | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | + | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | + +------------------+-------------+--------------------+-----------+----------+--------+------------+---------+-------+----------------------+------------+ + diff --git a/docs/experiment/ppl/index.rst b/docs/experiment/ppl/index.rst new file mode 100644 index 0000000000..6645d9e255 --- /dev/null +++ b/docs/experiment/ppl/index.rst @@ -0,0 +1,45 @@ + +=============================== +OpenDistro PPL Reference Manual +=============================== + +| Open Distro for Elasticsearch PPL enables you to extract insights out of Elasticsearch using the familiar pipe processing language query syntax. A PPL query is a read-only request to process data and return result. +| The query consists of a sequence of command, delimited by a pipe (|). The query start with search command and then flowing a set of command delimited by pipe (|). +| for example, the following query retrieve firstname and lastname from accounts if age large than 18. + +.. code-block:: + + source=accounts + | where age > 18 + | fields firstname, lastname + +* **Interfaces** + + - `Endpoint `_ + + - `Protocol `_ + +* **Administration** + + - `Plugin Settings `_ + +* **Commands** + + - `Syntax `_ + + - `dedup command `_ + + - `eval command `_ + + - `field command `_ + + - `rename command `_ + + - `search command `_ + + - `sort command `_ + + - `stats command `_ + + - `where command `_ + diff --git a/docs/experiment/ppl/interfaces/endpoint.rst b/docs/experiment/ppl/interfaces/endpoint.rst new file mode 100644 index 0000000000..45e34b28b1 --- /dev/null +++ b/docs/experiment/ppl/interfaces/endpoint.rst @@ -0,0 +1,67 @@ +.. highlight:: sh + +======== +Endpoint +======== + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 1 + + +Introduction +============ + +To send query request to PPL plugin, you MUST use HTTP POST request. POST request doesn't have length limitation and allows for other parameters passed to plugin for other functionality such as prepared statement. And also the explain endpoint is used very often for query translation and troubleshooting. + +POST +==== + +Description +----------- + +You can send HTTP POST request to endpoint **/_opendistro/_ppl** with your query in request body. + +Example +------- + +PPL query:: + + sh$ curl -sS -H 'Content-Type: application/json' \ + ... -X POST localhost:9200/_opendistro/_ppl \ + ... -d '{"query" : "source=accounts | fields firstname, lastname"}' + { + "schema": [ + { + "name": "firstname", + "type": "string" + }, + { + "name": "lastname", + "type": "string" + } + ], + "total": 4, + "datarows": [ + [ + "Amber", + "Duke" + ], + [ + "Hattie", + "Bond" + ], + [ + "Nanette", + "Bates" + ], + [ + "Dale", + "Adams" + ] + ], + "size": 4 + } + diff --git a/docs/experiment/ppl/interfaces/protocol.rst b/docs/experiment/ppl/interfaces/protocol.rst new file mode 100644 index 0000000000..05ed849fca --- /dev/null +++ b/docs/experiment/ppl/interfaces/protocol.rst @@ -0,0 +1,133 @@ +.. highlight:: sh + +======== +Protocol +======== + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 1 + + +Introduction +============ + +For the protocol, PPL endpoint provides response formats in the JDBC format. JDBC format is widely used because it provides schema information and more functionality such as pagination. Besides JDBC driver, various clients can benefit from the detailed and well formatted response. + + +Request/Response Format +============== + +Description +----------- + +The body of HTTP POST request can take PPL query. + +Example 1 +--------- +PPL query:: + + sh$ curl -sS -H 'Content-Type: application/json' \ + ... -X POST localhost:9200/_opendistro/_ppl \ + ... -d '{"query" : "source=accounts | fields firstname, lastname"}' + { + "schema": [ + { + "name": "firstname", + "type": "string" + }, + { + "name": "lastname", + "type": "string" + } + ], + "total": 4, + "datarows": [ + [ + "Amber", + "Duke" + ], + [ + "Hattie", + "Bond" + ], + [ + "Nanette", + "Bates" + ], + [ + "Dale", + "Adams" + ] + ], + "size": 4 + } + +JDBC Format +=========== + +Description +----------- + +By default the plugin return JDBC format. JDBC format is provided for JDBC driver and client side that needs both schema and result set well formatted. + +Example 1 +--------- + +Here is an example for normal response. The `schema` includes field name and its type and `datarows` includes the result set. + +PPL query:: + + sh$ curl -sS -H 'Content-Type: application/json' \ + ... -X POST localhost:9200/_opendistro/_ppl \ + ... -d '{"query" : "source=accounts | fields firstname, lastname"}' + { + "schema": [ + { + "name": "firstname", + "type": "string" + }, + { + "name": "lastname", + "type": "string" + } + ], + "total": 4, + "datarows": [ + [ + "Amber", + "Duke" + ], + [ + "Hattie", + "Bond" + ], + [ + "Nanette", + "Bates" + ], + [ + "Dale", + "Adams" + ] + ], + "size": 4 + } + +Example 2 +--------- + +If any error occurred, error message and the cause will be returned instead. + +PPL query:: + + sh$ curl -sS -H 'Content-Type: application/json' \ + ... -X POST localhost:9200/_opendistro/_ppl \ + ... -d '{"query" : "source=unknown | fields firstname, lastname"}' + { + "reason": "no such index [unknown]", + "type": "IndexNotFoundException" + } + diff --git a/docs/user/admin/settings.rst b/docs/user/admin/settings.rst index 82ba48a10f..2029549195 100644 --- a/docs/user/admin/settings.rst +++ b/docs/user/admin/settings.rst @@ -510,3 +510,47 @@ Result set:: } } +opendistro.sql.engine.new.enabled +================================= + +Description +----------- + +We are migrating existing functionalities to a new query engine under development. User can choose to enable the new engine if interested or disable if any issue found. + +1. The default value is false. +2. This setting is node scope. +3. This setting can be updated dynamically. + + +Example +------- + +You can update the setting with a new value like this. + +SQL query:: + + >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_opendistro/_sql/settings -d '{ + "transient" : { + "opendistro.sql.engine.new.enabled" : "true" + } + }' + +Result set:: + + { + "acknowledged" : true, + "persistent" : { }, + "transient" : { + "opendistro" : { + "sql" : { + "engine" : { + "new" : { + "enabled" : "true" + } + } + } + } + } + } + diff --git a/docs/user/dql/expressions.rst b/docs/user/dql/expressions.rst new file mode 100644 index 0000000000..c81f6ee6f7 --- /dev/null +++ b/docs/user/dql/expressions.rst @@ -0,0 +1,132 @@ +=========== +Expressions +=========== + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 3 + + +Introduction +============ + +Expressions, particularly value expressions, are those which return a scalar value. Expressions have different types and forms. For example, there are literal values as atom expression and arithmetic, predicate and function expression built on top of them. And also expressions can be used in different clauses, such as using arithmetic expression in ``SELECT``, ``WHERE`` or ``HAVING`` clause. + +Note that before you try out examples using the SQL features in this doc, you need to enable the new query engine by following the steps in ``opendistro.sql.engine.new.enabled`` section in `Plugin Settings `_. + +Literal Values +============== + +Description +----------- + +A literal is a symbol that represents a value. The most common literal values include: + +1. Numeric literals: specify numeric values such as integer and floating-point numbers. +2. String literals: specify a string enclosed by single or double quotes. +3. Boolean literals: ``true`` or ``false``. + +Examples +-------- + +Here is an example for different type of literals:: + + od> SELECT 123, 'hello', false, -4.567; + fetched rows / total rows = 1/1 + +-------+-----------+---------+----------+ + | 123 | "hello" | false | -4.567 | + |-------+-----------+---------+----------| + | 123 | hello | False | -4.567 | + +-------+-----------+---------+----------+ + +Limitations +----------- + +The current implementation has the following limitations at the moment: + +1. Only literals of data types listed as above are supported for now. Other type of literals, such as date and NULL, will be added in future. +2. Expression of literals, such as arithmetic expressions, will be supported later. +3. Standard ANSI ``VALUES`` clause is not supported, although the ``SELECT`` literal example above is implemented by a Values operator internally. + +Arithmetic Expressions +====================== + +Description +----------- + +Operators +````````` + +Arithmetic expression is an expression formed by numeric literals and binary arithmetic operators as follows: + +1. ``+``: Add. +2. ``-``: Subtract. +3. ``*``: Multiply. +4. ``/``: Divide. For integers, the result is an integer with fractional part discarded. +5. ``%``: Modulo. This can be used with integers only with remainder of the division as result. + +Precedence +`````````` + +Parentheses can be used to control the precedence of arithmetic operators. Otherwise, operators of higher precedence is performed first. + +Type Conversion +``````````````` + +Implicit type conversion is performed when looking up operator signature. For example, an integer ``+`` a real number matches signature ``+(double,double)`` which results in a real number. This rule also applies to function call discussed below. + +Examples +-------- + +Here is an example for different type of arithmetic expressions:: + + od> SELECT 1 + 2, (9 - 1) % 3, 2 * 4 / 3; + fetched rows / total rows = 1/1 + +---------+-------------+-------------+ + | 1 + 2 | 9 - 1 % 3 | 2 * 4 / 3 | + |---------+-------------+-------------| + | 3 | 2 | 2 | + +---------+-------------+-------------+ + + +Function Call +============= + +Description +----------- + +A function call is declared by function name followed by its arguments. The arguments are enclosed in parentheses and separated by comma. For complete function list supported, please see also: `SQL Functions `_ + +Syntax +`````` + +A typical function call is in the following form:: + + function_name ( [ expression [, expression]* ]? ) + +Null Handling +````````````` + +If any argument is missing or null, the final result of evaluation will be missing or null accordingly. + +Examples +-------- + +Here is an example for different type of arithmetic expressions:: + + od> SELECT abs(-1.234), abs(-1 * abs(-5)); + fetched rows / total rows = 1/1 + +---------------+---------------------+ + | abs(-1.234) | abs(-1 * abs(-5)) | + |---------------+---------------------| + | 1.234 | 5 | + +---------------+---------------------+ + +Limitations +----------- + +1. Only a subset of the SQL functions above is implemented in new engine for now. More function support are being added. +2. For now function name is required to be lowercase. + diff --git a/docs/user/dql/newsql.rst b/docs/user/dql/newsql.rst new file mode 100644 index 0000000000..9c7eed42c7 --- /dev/null +++ b/docs/user/dql/newsql.rst @@ -0,0 +1,35 @@ +.. highlight:: sh + +============== +New SQL Engine +============== + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + +Introduction +============ + +To use the SQL features present in documentation correctly, you need to enable our new SQL query engine by the following command:: + + sh$ curl -sS -H 'Content-Type: application/json' \ + ... -X PUT localhost:9200/_opendistro/_sql/settings \ + ... -d '{"transient" : {"opendistro.sql.engine.new.enabled" : "true"}}' + { + "acknowledged": true, + "persistent": {}, + "transient": { + "opendistro": { + "sql": { + "engine": { + "new": { + "enabled": "true" + } + } + } + } + } + } diff --git a/docs/user/index.rst b/docs/user/index.rst index 028e7b7e76..c3562c305c 100644 --- a/docs/user/index.rst +++ b/docs/user/index.rst @@ -19,6 +19,8 @@ Open Distro for Elasticsearch SQL enables you to extract insights out of Elastic * **Data Query Language** + - `Expressions `_ + - `Basic Queries `_ - `Complex Queries `_ diff --git a/docs/user/interfaces/endpoint.rst b/docs/user/interfaces/endpoint.rst index dafbe5808b..970f4f3887 100644 --- a/docs/user/interfaces/endpoint.rst +++ b/docs/user/interfaces/endpoint.rst @@ -14,7 +14,7 @@ Endpoint Introduction ============ -To send query request to SQL plugin, you can either use a request parameter in HTTP GET or request body by HTTP POST request. POST request is recommended because it doesn't have length limitation and allows for other parameters passed to plugin for other functionality such as prepared statement. And also the explain endpoint is used very often for query translation and troubleshooting. +To send query request to SQL plugin, you MUST use HTTP POST request. POST request doesn't have length limitation and allows for other parameters passed to plugin for other functionality such as prepared statement. And also the explain endpoint is used very often for query translation and troubleshooting. POST ==== diff --git a/doctest/bin/test-docs b/doctest/bin/test-docs new file mode 100755 index 0000000000..4dd4390b89 --- /dev/null +++ b/doctest/bin/test-docs @@ -0,0 +1,3 @@ +#!/bin/bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +$DIR/../.venv/bin/python -X faulthandler -m unittest -v --failfast test_docs diff --git a/doctest/bootstrap.sh b/doctest/bootstrap.sh new file mode 100755 index 0000000000..2bf247e516 --- /dev/null +++ b/doctest/bootstrap.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +DIR=$(dirname "$0") + +if hash python3.7 2> /dev/null; then + PYTHON=python3.7 +elif hash python3 2> /dev/null; then + # fallback to python3 in case there is no python3.7 alias; should be 3.7 + PYTHON=python3 +else + echo 'python3.7 required' + exit 1 +fi + +$PYTHON -m venv $DIR/.venv +if [ ! -f $DIR/.venv/bin/pip ]; then + wget https://bootstrap.pypa.io/get-pip.py + $DIR/.venv/bin/python get-pip.py + rm -f get-pip.py +fi + +$DIR/.venv/bin/pip install -U pip setuptools wheel +$DIR/.venv/bin/pip install -r $DIR/requirements.txt +# Temporary fix, add odfe-sql-cli dependency into requirements.txt once we have released cli to PyPI +$DIR/.venv/bin/pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple odfe-sql-cli==0.0.1 diff --git a/doctest/build.gradle b/doctest/build.gradle new file mode 100644 index 0000000000..5d42fe6229 --- /dev/null +++ b/doctest/build.gradle @@ -0,0 +1,39 @@ +plugins { + id "com.wiredforcode.spawn" version "0.8.2" + id 'base' +} + +def path = project(':').projectDir +// temporary fix, because currently we are under migration to new architecture. Need to run ./gradlew run from +// plugin module, and will only build ppl in it. +def plugin_path = project(':plugin').projectDir + +task bootstrap(type: Exec) { + inputs.file "$projectDir/bootstrap.sh" + outputs.dir "$projectDir/.venv" + + commandLine 'sh', "$projectDir/bootstrap.sh" +} + +//evaluationDependsOn(':') +task startES(type: SpawnProcessTask) { + command "${path}/gradlew -p ${plugin_path} run" + ready 'started' +} + +task doctest(type: Exec, dependsOn: ['bootstrap']) { + + commandLine "$projectDir/bin/test-docs" + + doLast { + println("Doctest Done") + } +} + +task stopES(type: KillProcessTask) + +doctest.dependsOn startES +doctest.finalizedBy stopES + +build.dependsOn doctest +clean.dependsOn(cleanBootstrap) \ No newline at end of file diff --git a/doctest/requirements.txt b/doctest/requirements.txt new file mode 100644 index 0000000000..6d84269eee --- /dev/null +++ b/doctest/requirements.txt @@ -0,0 +1,2 @@ +zc.customdoctests==1.0.1 +# Add odfe-sql-cli once we have it on official PyPI \ No newline at end of file diff --git a/doctest/test_data/accounts.json b/doctest/test_data/accounts.json new file mode 100644 index 0000000000..2ea2b602fd --- /dev/null +++ b/doctest/test_data/accounts.json @@ -0,0 +1,4 @@ +{"account_number":1,"balance":39225,"firstname":"Amber","lastname":"Duke","age":32,"gender":"M","address":"880 Holmes Lane","employer":"Pyrami","email":"amberduke@pyrami.com","city":"Brogan","state":"IL"} +{"account_number":6,"balance":5686,"firstname":"Hattie","lastname":"Bond","age":36,"gender":"M","address":"671 Bristol Street","employer":"Netagy","email":"hattiebond@netagy.com","city":"Dante","state":"TN"} +{"account_number":13,"balance":32838,"firstname":"Nanette","lastname":"Bates","age":28,"gender":"F","address":"789 Madison Street","employer":"Quility","email":null,"city":"Nogal","state":"VA"} +{"account_number":18,"balance":4180,"firstname":"Dale","lastname":"Adams","age":33,"gender":"M","address":"467 Hutchinson Court","employer":null,"email":"daleadams@boink.com","city":"Orick","state":"MD"} diff --git a/doctest/test_docs.py b/doctest/test_docs.py new file mode 100644 index 0000000000..05829addbb --- /dev/null +++ b/doctest/test_docs.py @@ -0,0 +1,201 @@ +# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +import doctest +import os +import zc.customdoctests +import json +import re +import random +import subprocess +import unittest +import click + +from functools import partial +from odfe_sql_cli.esconnection import ESConnection +from odfe_sql_cli.utils import OutputSettings +from odfe_sql_cli.formatter import Formatter +from elasticsearch import Elasticsearch, helpers + +ENDPOINT = "http://localhost:9200" +ACCOUNTS = "accounts" +EMPLOYEES = "employees" + + +class DocTestConnection(ESConnection): + + def __init__(self, query_language="sql"): + super(DocTestConnection, self).__init__(endpoint=ENDPOINT, query_language=query_language) + self.set_connection() + + settings = OutputSettings(table_format="psql", is_vertical=False) + self.formatter = Formatter(settings) + + def process(self, statement): + data = self.execute_query(statement, use_console=False) + output = self.formatter.format_output(data) + output = "\n".join(output) + + click.echo(output) + + +def pretty_print(s): + try: + d = json.loads(s) + print(json.dumps(d, indent=2)) + except json.decoder.JSONDecodeError: + print(s) + + +sql_cmd = DocTestConnection(query_language="sql") +ppl_cmd = DocTestConnection(query_language="ppl") +test_data_client = Elasticsearch([ENDPOINT], verify_certs=True) + + +def sql_cli_transform(s): + return u'sql_cmd.process({0})'.format(repr(s.strip().rstrip(';'))) + + +def ppl_cli_transform(s): + return u'ppl_cmd.process({0})'.format(repr(s.strip().rstrip(';'))) + + +def bash_transform(s): + # TODO: add ppl support, be default cli uses sql + if s.startswith("odfesql"): + s = re.search(r"odfesql\s+-q\s+\"(.*?)\"", s).group(1) + return u'cmd.process({0})'.format(repr(s.strip().rstrip(';'))) + return (r'pretty_print(sh("""%s""").stdout.decode("utf-8"))' % s) + '\n' + + +sql_cli_parser = zc.customdoctests.DocTestParser( + ps1='od>', comment_prefix='#', transform=sql_cli_transform) + +ppl_cli_parser = zc.customdoctests.DocTestParser( + ps1='od>', comment_prefix='#', transform=ppl_cli_transform) + +bash_parser = zc.customdoctests.DocTestParser( + ps1=r'sh\$', comment_prefix='#', transform=bash_transform) + + +def set_up_accounts(test): + set_up(test) + load_file("accounts.json", index_name=ACCOUNTS) + + +def load_file(filename, index_name): + filepath = "./test_data/" + filename + + # generate iterable data + def load_json(): + with open(filepath, "r") as f: + for line in f: + yield json.loads(line) + + # Need to enable refresh, because the load won't be visible to search immediately + # https://stackoverflow.com/questions/57840161/elasticsearch-python-bulk-helper-api-with-refresh + helpers.bulk(test_data_client, load_json(), stats_only=True, index=index_name, refresh='wait_for') + + +def set_up(test): + test.globs['sql_cmd'] = sql_cmd + test.globs['ppl_cmd'] = ppl_cmd + + +def tear_down(test): + # drop leftover tables after each test + test_data_client.indices.delete(index=[ACCOUNTS, EMPLOYEES], ignore_unavailable=True) + + +docsuite = partial(doctest.DocFileSuite, + tearDown=tear_down, + parser=sql_cli_parser, + optionflags=doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS, + encoding='utf-8') + + +doctest_file = partial(os.path.join, '../docs') + + +def doctest_files(items): + return (doctest_file(item) for item in items) + + +class DocTests(unittest.TestSuite): + def run(self, result, debug=False): + super().run(result, debug) + + +def doc_suite(fn): + return docsuite( + fn, + parser=bash_parser, + setUp=set_up_accounts, + globs={ + 'sh': partial( + subprocess.run, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=60, + shell=True + ), + 'pretty_print': pretty_print + } + ) + + +def load_tests(loader, suite, ignore): + tests = [] + # Load doctest docs by category + with open('../docs/category.json') as json_file: + category = json.load(json_file) + + bash_docs = category['bash'] + ppl_cli_docs = category['ppl_cli'] + sql_cli_docs = category['sql_cli'] + + # docs with bash-based examples + for fn in doctest_files(bash_docs): + tests.append(doc_suite(fn)) + + # docs with sql-cli based examples + # TODO: add until the migration to new architecture is done, then we have an artifact including ppl and sql both + # for fn in doctest_files('sql/basics.rst'): + # tests.append(docsuite(fn, setUp=set_up_accounts)) + for fn in doctest_files(sql_cli_docs): + tests.append( + docsuite( + fn, + parser=sql_cli_parser, + setUp=set_up_accounts + ) + ) + + # docs with ppl-cli based examples + for fn in doctest_files(ppl_cli_docs): + tests.append( + docsuite( + fn, + parser=ppl_cli_parser, + setUp=set_up_accounts + ) + ) + + # randomize order of tests to make sure they don't depend on each other + random.shuffle(tests) + + # prepend a temporary doc to enable new engine so new SQL docs followed can pass + tests.insert(0, doc_suite('../docs/user/dql/newsql.rst')) + + return DocTests(tests) diff --git a/elasticsearch/build.gradle b/elasticsearch/build.gradle new file mode 100644 index 0000000000..b8407262ef --- /dev/null +++ b/elasticsearch/build.gradle @@ -0,0 +1,65 @@ +plugins { + id 'java' + id "io.freefair.lombok" + id 'jacoco' +} + +repositories { + mavenCentral() +} + +dependencies { + compile project(':core') + compile group: 'org.elasticsearch', name: 'elasticsearch', version: "${es_version}" + compile group: 'org.elasticsearch.client', name: 'elasticsearch-rest-high-level-client', version: "${es_version}" + compile "io.github.resilience4j:resilience4j-retry:1.5.0" + + testImplementation('org.junit.jupiter:junit-jupiter:5.6.2') + testCompile group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' + testCompile group: 'org.mockito', name: 'mockito-core', version: '3.3.3' + testCompile group: 'org.mockito', name: 'mockito-junit-jupiter', version: '3.3.3' +} + +test { + useJUnitPlatform() + testLogging { + events "passed", "skipped", "failed" + } +} + +jacoco { + toolVersion = "0.8.5" +} +jacocoTestReport { + reports { + html.enabled true + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it) + })) + } +} +test.finalizedBy(project.tasks.jacocoTestReport) + +jacocoTestCoverageVerification { + violationRules { + rule { + element = 'CLASS' + excludes = [ + 'com.amazon.opendistroforelasticsearch.sql.elasticsearch.security.SecurityAccess' + ] + limit { + minimum = 1.0 + } + + } + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it) + })) + } +} +check.dependsOn jacocoTestCoverageVerification +jacocoTestCoverageVerification.dependsOn jacocoTestReport \ No newline at end of file diff --git a/elasticsearch/lombok.config b/elasticsearch/lombok.config new file mode 100644 index 0000000000..189c0bef98 --- /dev/null +++ b/elasticsearch/lombok.config @@ -0,0 +1,3 @@ +# This file is generated by the 'io.freefair.lombok' Gradle plugin +config.stopBubbling = true +lombok.addLombokGeneratedAnnotation = true diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchClient.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchClient.java new file mode 100644 index 0000000000..13a82ac751 --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchClient.java @@ -0,0 +1,59 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.client; + +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.mapping.IndexMapping; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.request.ElasticsearchRequest; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.response.ElasticsearchResponse; +import java.util.Map; + +/** + * Elasticsearch client abstraction to wrap different Elasticsearch client implementation. For + * example, implementation by node client for ES plugin or by REST client for standalone mode. + */ +public interface ElasticsearchClient { + + /** + * Fetch index mapping(s) according to index expression given. + * + * @param indexExpression index expression + * @return index mapping(s) from index name to its mapping + */ + Map getIndexMappings(String indexExpression); + + /** + * Perform search query in the search request. + * + * @param request search request + * @return search response + */ + ElasticsearchResponse search(ElasticsearchRequest request); + + /** + * Clean up resources related to the search request, for example scroll context. + * + * @param request search request + */ + void cleanup(ElasticsearchRequest request); + + /** + * Schedule a task to run. + * + * @param task task + */ + void schedule(Runnable task); +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchNodeClient.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchNodeClient.java new file mode 100644 index 0000000000..b5df2af5cf --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchNodeClient.java @@ -0,0 +1,151 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.client; + +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.mapping.IndexMapping; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.request.ElasticsearchRequest; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.response.ElasticsearchResponse; +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import java.util.function.Function; +import java.util.function.Predicate; +import lombok.RequiredArgsConstructor; +import org.apache.logging.log4j.ThreadContext; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.threadpool.ThreadPool; + +/** Elasticsearch connection by node client. */ +@RequiredArgsConstructor +public class ElasticsearchNodeClient implements ElasticsearchClient { + + /** Default types and field filter to match all. */ + public static final String[] ALL_TYPES = new String[0]; + + public static final Function> ALL_FIELDS = + (anyIndex -> (anyField -> true)); + + /** Current cluster state on local node. */ + private final ClusterService clusterService; + + /** Node client provided by Elasticsearch container. */ + private final NodeClient client; + + /** Index name expression resolver to get concrete index name. */ + private final IndexNameExpressionResolver resolver = new IndexNameExpressionResolver(); + + private static final String SQL_WORKER_THREAD_POOL_NAME = "sql-worker"; + + /** + * Get field mappings of index by an index expression. Majority is copied from legacy + * LocalClusterState. + * + *

For simplicity, removed type (deprecated) and field filter in argument list. Also removed + * mapping cache, cluster state listener (mainly for performance and debugging). + * + * @param indexExpression index name expression + * @return index mapping(s) in our class to isolate Elasticsearch API. IndexNotFoundException is + * thrown if no index matched. + */ + @Override + public Map getIndexMappings(String indexExpression) { + try { + ClusterState state = clusterService.state(); + String[] concreteIndices = resolveIndexExpression(state, new String[] {indexExpression}); + + return populateIndexMappings( + state.metadata().findMappings(concreteIndices, ALL_TYPES, ALL_FIELDS)); + } catch (IOException e) { + throw new IllegalStateException( + "Failed to read mapping in cluster state for index pattern [" + indexExpression + "]", e); + } + } + + /** TODO: Scroll doesn't work for aggregation. Support aggregation later. */ + @Override + public ElasticsearchResponse search(ElasticsearchRequest request) { + SearchResponse esResponse; + if (request.isScrollStarted()) { + esResponse = client.searchScroll(request.scrollRequest()).actionGet(); + } else { + esResponse = client.search(request.searchRequest()).actionGet(); + } + request.setScrollId(esResponse.getScrollId()); + + return new ElasticsearchResponse(esResponse); + } + + @Override + public void cleanup(ElasticsearchRequest request) { + if (request.isScrollStarted()) { + client.prepareClearScroll().addScrollId(request.getScrollId()).get(); + request.reset(); + } + } + + @Override + public void schedule(Runnable task) { + ThreadPool threadPool = client.threadPool(); + threadPool.schedule( + withCurrentContext(task), + new TimeValue(0), + SQL_WORKER_THREAD_POOL_NAME + ); + } + + private String[] resolveIndexExpression(ClusterState state, String[] indices) { + return resolver.concreteIndexNames(state, IndicesOptions.strictExpandOpen(), indices); + } + + private Map populateIndexMappings( + ImmutableOpenMap> indexMappings) { + + ImmutableMap.Builder result = ImmutableMap.builder(); + for (ObjectObjectCursor> cursor : + indexMappings) { + result.put(cursor.key, populateIndexMapping(cursor.value)); + } + return result.build(); + } + + private IndexMapping populateIndexMapping( + ImmutableOpenMap indexMapping) { + if (indexMapping.isEmpty()) { + return new IndexMapping(Collections.emptyMap()); + } + return new IndexMapping(indexMapping.iterator().next().value); + } + + /** Copy from LogUtils. */ + private static Runnable withCurrentContext(final Runnable task) { + final Map currentContext = ThreadContext.getImmutableContext(); + return () -> { + ThreadContext.putAll(currentContext); + task.run(); + }; + } +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchRestClient.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchRestClient.java new file mode 100644 index 0000000000..ec66dbca75 --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchRestClient.java @@ -0,0 +1,96 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.client; + +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.mapping.IndexMapping; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.request.ElasticsearchRequest; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.response.ElasticsearchResponse; +import java.io.IOException; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import org.elasticsearch.action.search.ClearScrollRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.indices.GetMappingsRequest; +import org.elasticsearch.client.indices.GetMappingsResponse; + +/** + * Elasticsearch REST client to support standalone mode that runs entire engine from remote. + * + *

TODO: Support for authN and authZ with AWS Sigv4 or security plugin. + */ +@RequiredArgsConstructor +public class ElasticsearchRestClient implements ElasticsearchClient { + + /** Elasticsearch high level REST client. */ + private final RestHighLevelClient client; + + @Override + public Map getIndexMappings(String indexExpression) { + GetMappingsRequest request = new GetMappingsRequest().indices(indexExpression); + try { + GetMappingsResponse response = client.indices().getMapping(request, RequestOptions.DEFAULT); + return response.mappings().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> new IndexMapping(e.getValue()))); + } catch (IOException e) { + throw new IllegalStateException("Failed to get index mappings for " + indexExpression, e); + } + } + + @Override + public ElasticsearchResponse search(ElasticsearchRequest request) { + try { + SearchResponse esResponse; + if (request.isScrollStarted()) { + esResponse = client.scroll(request.scrollRequest(), RequestOptions.DEFAULT); + } else { + esResponse = client.search(request.searchRequest(), RequestOptions.DEFAULT); + } + request.setScrollId(esResponse.getScrollId()); + + return new ElasticsearchResponse(esResponse); + } catch (IOException e) { + throw new IllegalStateException( + "Failed to perform search operation with request " + request, e); + } + } + + @Override + public void cleanup(ElasticsearchRequest request) { + try { + if (!request.isScrollStarted()) { + return; + } + + ClearScrollRequest clearRequest = new ClearScrollRequest(); + clearRequest.addScrollId(request.getScrollId()); + client.clearScroll(clearRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + throw new IllegalStateException( + "Failed to clean up resources for search request " + request, e); + } finally { + request.reset(); + } + } + + @Override + public void schedule(Runnable task) { + task.run(); + } +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/ElasticsearchExecutionEngine.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/ElasticsearchExecutionEngine.java new file mode 100644 index 0000000000..b3caa33b83 --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/ElasticsearchExecutionEngine.java @@ -0,0 +1,59 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor; + +import com.amazon.opendistroforelasticsearch.sql.common.response.ResponseListener; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor.protector.ExecutionProtector; +import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import java.util.ArrayList; +import java.util.List; +import lombok.RequiredArgsConstructor; + +/** Elasticsearch execution engine implementation. */ +@RequiredArgsConstructor +public class ElasticsearchExecutionEngine implements ExecutionEngine { + + private final ElasticsearchClient client; + + private final ExecutionProtector executionProtector; + + @Override + public void execute(PhysicalPlan physicalPlan, ResponseListener listener) { + PhysicalPlan plan = executionProtector.protect(physicalPlan); + client.schedule( + () -> { + try { + List result = new ArrayList<>(); + plan.open(); + + while (plan.hasNext()) { + result.add(plan.next()); + } + + QueryResponse response = new QueryResponse(result); + listener.onResponse(response); + } catch (Exception e) { + listener.onFailure(e); + } finally { + plan.close(); + } + }); + } +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/protector/ElasticsearchExecutionProtector.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/protector/ElasticsearchExecutionProtector.java new file mode 100644 index 0000000000..db82cb3b7d --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/protector/ElasticsearchExecutionProtector.java @@ -0,0 +1,107 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor.protector; + +import com.amazon.opendistroforelasticsearch.sql.monitor.ResourceMonitor; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.AggregationOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.DedupeOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.EvalOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.FilterOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlanNodeVisitor; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.ProjectOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.RemoveOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.RenameOperator; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.SortOperator; +import com.amazon.opendistroforelasticsearch.sql.storage.TableScanOperator; +import lombok.RequiredArgsConstructor; + +/** + * Elasticsearch Execution Protector. + */ +@RequiredArgsConstructor +public class ElasticsearchExecutionProtector extends ExecutionProtector { + + /** + * Elasticsearch resource monitor. + */ + private final ResourceMonitor resourceMonitor; + + public PhysicalPlan protect(PhysicalPlan physicalPlan) { + return physicalPlan.accept(this, null); + } + + @Override + public PhysicalPlan visitFilter(FilterOperator node, Object context) { + return new FilterOperator(visitInput(node.getInput(), context), node.getConditions()); + } + + @Override + public PhysicalPlan visitAggregation(AggregationOperator node, Object context) { + return new AggregationOperator(visitInput(node.getInput(), context), node.getAggregatorList(), + node.getGroupByExprList()); + } + + @Override + public PhysicalPlan visitRename(RenameOperator node, Object context) { + return new RenameOperator(visitInput(node.getInput(), context), node.getMapping()); + } + + /** + * Decorate with {@link ResourceMonitorPlan}. + */ + @Override + public PhysicalPlan visitTableScan(TableScanOperator node, Object context) { + return new ResourceMonitorPlan(node, resourceMonitor); + } + + @Override + public PhysicalPlan visitProject(ProjectOperator node, Object context) { + return new ProjectOperator(visitInput(node.getInput(), context), node.getProjectList()); + } + + @Override + public PhysicalPlan visitRemove(RemoveOperator node, Object context) { + return new RemoveOperator(visitInput(node.getInput(), context), node.getRemoveList()); + } + + @Override + public PhysicalPlan visitEval(EvalOperator node, Object context) { + return new EvalOperator(visitInput(node.getInput(), context), node.getExpressionList()); + } + + @Override + public PhysicalPlan visitDedupe(DedupeOperator node, Object context) { + return new DedupeOperator(visitInput(node.getInput(), context), node.getDedupeList(), + node.getAllowedDuplication(), node.getKeepEmpty(), node.getConsecutive()); + } + + @Override + public PhysicalPlan visitSort(SortOperator node, Object context) { + return new SortOperator(visitInput(node.getInput(), context), node.getCount(), + node.getSortList()); + } + + PhysicalPlan visitInput(PhysicalPlan node, Object context) { + if (null == node) { + return node; + } else { + return node.accept(this, context); + } + } +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/protector/ExecutionProtector.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/protector/ExecutionProtector.java new file mode 100644 index 0000000000..81b5d1cb23 --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/protector/ExecutionProtector.java @@ -0,0 +1,32 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor.protector; + +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlanNodeVisitor; + +/** + * Execution Plan Protector. + */ +public abstract class ExecutionProtector extends PhysicalPlanNodeVisitor { + + /** + * Decorated the PhysicalPlan to run in resource sensitive mode. + */ + public abstract PhysicalPlan protect(PhysicalPlan physicalPlan); +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/protector/NoopExecutionProtector.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/protector/NoopExecutionProtector.java new file mode 100644 index 0000000000..6aa656db67 --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/protector/NoopExecutionProtector.java @@ -0,0 +1,31 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor.protector; + +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; + +/** + * No operation execution protector. + */ +public class NoopExecutionProtector extends ExecutionProtector { + + @Override + public PhysicalPlan protect(PhysicalPlan physicalPlan) { + return physicalPlan; + } +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/protector/ResourceMonitorPlan.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/protector/ResourceMonitorPlan.java new file mode 100644 index 0000000000..0225a46974 --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/protector/ResourceMonitorPlan.java @@ -0,0 +1,79 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor.protector; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.monitor.ResourceMonitor; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlanNodeVisitor; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * A PhysicalPlan which will run the delegate plan in resource protection manner. + */ +@ToString +@RequiredArgsConstructor +@EqualsAndHashCode +public class ResourceMonitorPlan extends PhysicalPlan { + /** + * Delegated PhysicalPlan. + */ + private final PhysicalPlan delegate; + + /** + * ResourceMonitor. + */ + @ToString.Exclude + private final ResourceMonitor monitor; + + @Override + public R accept(PhysicalPlanNodeVisitor visitor, C context) { + return delegate.accept(visitor, context); + } + + @Override + public void open() { + if (!this.monitor.isHealthy()) { + throw new IllegalStateException("resource is not enough to run the query, quit."); + } + delegate.open(); + } + + @Override + public void close() { + delegate.close(); + } + + @Override + public List getChild() { + return delegate.getChild(); + } + + @Override + public boolean hasNext() { + return delegate.hasNext(); + } + + @Override + public ExprValue next() { + return delegate.next(); + } +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/mapping/IndexMapping.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/mapping/IndexMapping.java new file mode 100644 index 0000000000..db176f1975 --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/mapping/IndexMapping.java @@ -0,0 +1,113 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.mapping; + +import static java.util.Collections.emptyMap; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.stream.Collectors; +import lombok.ToString; +import org.elasticsearch.cluster.metadata.MappingMetadata; + +/** + * Elasticsearch index mapping. Because there is no specific behavior for different field types, + * string is used to represent field types. + */ +@ToString +public class IndexMapping { + + /** Field mappings from field name to field type in Elasticsearch date type system. */ + private final Map fieldMappings; + + public IndexMapping(Map fieldMappings) { + this.fieldMappings = fieldMappings; + } + + public IndexMapping(MappingMetadata metaData) { + this.fieldMappings = flatMappings(metaData.getSourceAsMap()); + } + + /** + * How many fields in the index (after flatten). + * + * @return field size + */ + public int size() { + return fieldMappings.size(); + } + + /** + * Return field type by its name. + * + * @param fieldName field name + * @return field type in string. Or null if not exist. + */ + public String getFieldType(String fieldName) { + return fieldMappings.get(fieldName); + } + + /** + * Get all field types and transform raw string type to expected type. + * + * @param transform transform function to transform field type in string to another type + * @param expected field type class + * @return mapping from field name to field type + */ + public Map getAllFieldTypes(Function transform) { + return fieldMappings.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> transform.apply(e.getValue()))); + } + + @SuppressWarnings("unchecked") + private Map flatMappings(Map indexMapping) { + ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + + flatMappings( + ((Map) indexMapping.getOrDefault("properties", emptyMap())), + "", + builder::put); + return builder.build(); + } + + @SuppressWarnings("unchecked") + private void flatMappings( + Map mappings, String path, BiConsumer func) { + mappings.forEach( + (fieldName, mappingObject) -> { + Map mapping = (Map) mappingObject; + String fullFieldName = path.isEmpty() ? fieldName : path + "." + fieldName; + String type = (String) mapping.getOrDefault("type", "object"); + func.accept(fullFieldName, type); + + if (mapping.containsKey("fields")) { // Multi-field + ((Map>) mapping.get("fields")) + .forEach( + (innerFieldName, innerMapping) -> + func.accept( + fullFieldName + "." + innerFieldName, + (String) innerMapping.getOrDefault("type", "object"))); + } + + if (mapping.containsKey("properties")) { // Nested field + flatMappings((Map) mapping.get("properties"), fullFieldName, func); + } + }); + } +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/monitor/ElasticsearchMemoryHealthy.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/monitor/ElasticsearchMemoryHealthy.java new file mode 100644 index 0000000000..8a3e287c29 --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/monitor/ElasticsearchMemoryHealthy.java @@ -0,0 +1,86 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.monitor; + +import com.google.common.annotations.VisibleForTesting; +import java.util.concurrent.ThreadLocalRandom; +import lombok.NoArgsConstructor; +import lombok.extern.log4j.Log4j2; + +/** + * Elasticsearch Memory Monitor. + */ +@Log4j2 +public class ElasticsearchMemoryHealthy { + private final RandomFail randomFail; + private final MemoryUsage memoryUsage; + + public ElasticsearchMemoryHealthy() { + randomFail = new RandomFail(); + memoryUsage = new MemoryUsage(); + } + + @VisibleForTesting + public ElasticsearchMemoryHealthy( + RandomFail randomFail, + MemoryUsage memoryUsage) { + this.randomFail = randomFail; + this.memoryUsage = memoryUsage; + } + + /** + * Is Memory Healthy. Calculate based on the current heap memory usage. + */ + public boolean isMemoryHealthy(long limitBytes) { + final long memoryUsage = this.memoryUsage.usage(); + log.debug("Memory usage:{}, limit:{}", memoryUsage, limitBytes); + if (memoryUsage < limitBytes) { + return true; + } else { + log.warn("Memory usage:{} exceed limit:{}", memoryUsage, limitBytes); + if (randomFail.shouldFail()) { + log.warn("Fast failure the current request"); + throw new MemoryUsageExceedFastFailureException(); + } else { + throw new MemoryUsageExceedException(); + } + } + } + + static class RandomFail { + public boolean shouldFail() { + return ThreadLocalRandom.current().nextBoolean(); + } + } + + static class MemoryUsage { + public long usage() { + final long freeMemory = Runtime.getRuntime().freeMemory(); + final long totalMemory = Runtime.getRuntime().totalMemory(); + return totalMemory - freeMemory; + } + } + + @NoArgsConstructor + public static class MemoryUsageExceedFastFailureException extends RuntimeException { + + } + + @NoArgsConstructor + public static class MemoryUsageExceedException extends RuntimeException { + + } +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/monitor/ElasticsearchResourceMonitor.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/monitor/ElasticsearchResourceMonitor.java new file mode 100644 index 0000000000..980c833648 --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/monitor/ElasticsearchResourceMonitor.java @@ -0,0 +1,75 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.monitor; + +import com.amazon.opendistroforelasticsearch.sql.common.setting.Settings; +import com.amazon.opendistroforelasticsearch.sql.monitor.ResourceMonitor; +import io.github.resilience4j.core.IntervalFunction; +import io.github.resilience4j.retry.Retry; +import io.github.resilience4j.retry.RetryConfig; +import java.util.function.Supplier; +import lombok.extern.log4j.Log4j2; +import org.elasticsearch.common.unit.ByteSizeValue; + +/** + * {@link ResourceMonitor} implementation on Elasticsearch. When the heap memory usage exceeds + * certain threshold, the monitor is not healthy. + * Todo, add metrics. + */ +@Log4j2 +public class ElasticsearchResourceMonitor extends ResourceMonitor { + private final Settings settings; + private final Retry retry; + private final ElasticsearchMemoryHealthy memoryMonitor; + + /** + * Constructor of ElasticsearchCircuitBreaker. + */ + public ElasticsearchResourceMonitor( + Settings settings, + ElasticsearchMemoryHealthy memoryMonitor) { + this.settings = settings; + RetryConfig config = + RetryConfig.custom() + .maxAttempts(3) + .intervalFunction(IntervalFunction.ofExponentialRandomBackoff(1000)) + .retryExceptions(ElasticsearchMemoryHealthy.MemoryUsageExceedException.class) + .ignoreExceptions( + ElasticsearchMemoryHealthy.MemoryUsageExceedFastFailureException.class) + .build(); + retry = Retry.of("mem", config); + this.memoryMonitor = memoryMonitor; + } + + /** + * Is Healthy. + * + * @return true if healthy, otherwise return false. + */ + @Override + public boolean isHealthy() { + try { + ByteSizeValue limit = settings.getSettingValue(Settings.Key.PPL_QUERY_MEMORY_LIMIT); + Supplier booleanSupplier = + Retry.decorateSupplier(retry, + () -> memoryMonitor + .isMemoryHealthy(limit.getBytes())); + return booleanSupplier.get(); + } catch (Exception e) { + return false; + } + } +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/ElasticsearchRequest.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/ElasticsearchRequest.java new file mode 100644 index 0000000000..7f2a151044 --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/ElasticsearchRequest.java @@ -0,0 +1,95 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.request; + +import java.util.Objects; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchScrollRequest; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.search.builder.SearchSourceBuilder; + +/** + * Elasticsearch search request. This has to be stateful because it needs to: + * + *

1) Accumulate search source builder when visiting logical plan to push down operation 2) + * Maintain scroll ID between calls to client search method + */ +@EqualsAndHashCode +@RequiredArgsConstructor +@Getter +@ToString +public class ElasticsearchRequest { + + /** Default scroll context timeout in minutes. */ + public static final TimeValue DEFAULT_SCROLL_TIMEOUT = TimeValue.timeValueMinutes(1L); + + /** Index name. */ + private final String indexName; + + /** + * Scroll id which is set after first request issued. Because ElasticsearchClient is shared by + * multi-thread so this state has to be maintained here. + */ + @Setter private String scrollId; + + /** Search request source builder. */ + private final SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + + /** + * Generate Elasticsearch search request. + * + * @return search request + */ + public SearchRequest searchRequest() { + return new SearchRequest() + .indices(indexName) + .scroll(DEFAULT_SCROLL_TIMEOUT) + .source(sourceBuilder); + } + + /** + * Is scroll started which means pages after first is being requested. + * + * @return true if scroll started + */ + public boolean isScrollStarted() { + return (scrollId != null); + } + + /** + * Generate Elasticsearch scroll request by scroll id maintained. + * + * @return scroll request + */ + public SearchScrollRequest scrollRequest() { + Objects.requireNonNull(scrollId, "Scroll id cannot be null"); + return new SearchScrollRequest().scroll(DEFAULT_SCROLL_TIMEOUT).scrollId(scrollId); + } + + /** + * Reset internal state in case any stale data. However, ideally the same instance is not supposed + * to be reused across different physical plan. + */ + public void reset() { + scrollId = null; + } +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/response/ElasticsearchResponse.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/response/ElasticsearchResponse.java new file mode 100644 index 0000000000..e3382799ff --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/response/ElasticsearchResponse.java @@ -0,0 +1,57 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.response; + +import java.util.Iterator; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; + +/** Elasticsearch search response. */ +@EqualsAndHashCode +@ToString +public class ElasticsearchResponse implements Iterable { + + /** Search query result (non-aggregation). */ + private final SearchHits hits; + + public ElasticsearchResponse(SearchResponse esResponse) { + this.hits = esResponse.getHits(); // TODO: aggregation result is separate and not in SearchHit[] + } + + /** + * Is response empty. As ES doc says, "Each call to the scroll API returns the next batch of + * results until there are no more results left to return, ie the hits array is empty." + * + * @return true for empty + */ + public boolean isEmpty() { + return (hits.getHits() == null) || (hits.getHits().length == 0); + } + + /** + * Make response iterable without need to return internal data structure explicitly. + * + * @return search hit iterator + */ + @Override + public Iterator iterator() { + return hits.iterator(); + } +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/security/SecurityAccess.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/security/SecurityAccess.java new file mode 100644 index 0000000000..f42d100393 --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/security/SecurityAccess.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.security; + +import java.io.IOException; +import java.security.AccessController; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import org.elasticsearch.SpecialPermission; + +/** + * Ref: + * https://www.elastic.co/guide/en/elasticsearch/plugins/current/plugin-authors.html#_java_security_permissions + */ +public class SecurityAccess { + + /** + * Execute the operation in privileged mode. + */ + public static T doPrivileged(final PrivilegedExceptionAction operation) + throws IOException { + SpecialPermission.check(); + try { + return AccessController.doPrivileged(operation); + } catch (final PrivilegedActionException e) { + throw (IOException) e.getCause(); + } + } +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/setting/ElasticsearchSettings.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/setting/ElasticsearchSettings.java new file mode 100644 index 0000000000..da3d463592 --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/setting/ElasticsearchSettings.java @@ -0,0 +1,107 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.setting; + +import static org.elasticsearch.common.settings.Settings.EMPTY; + +import com.amazon.opendistroforelasticsearch.sql.common.setting.Settings; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Consumer; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Setting; + +/** + * Setting implementation on Elasticsearch. + */ +@Log4j2 +public class ElasticsearchSettings extends Settings { + /** + * Default settings. + */ + private final Map> defaultSettings; + /** + * Latest setting value for each registered key. Thread-safe is required. + */ + @VisibleForTesting + private final Map latestSettings = new ConcurrentHashMap<>(); + + private static final Setting PPL_QUERY_MEMORY_LIMIT_SETTINGS = Setting.memorySizeSetting( + Key.PPL_QUERY_MEMORY_LIMIT.getKeyValue(), + "85%", + Setting.Property.NodeScope, + Setting.Property.Dynamic); + + /** + * Construct ElasticsearchSetting. + * The ElasticsearchSetting must be singleton. + */ + public ElasticsearchSettings(ClusterSettings clusterSettings) { + ImmutableMap.Builder> settingBuilder = new ImmutableMap.Builder<>(); + register(settingBuilder, clusterSettings, Key.PPL_QUERY_MEMORY_LIMIT, + PPL_QUERY_MEMORY_LIMIT_SETTINGS, new Updater(Key.PPL_QUERY_MEMORY_LIMIT)); + defaultSettings = settingBuilder.build(); + } + + @SuppressWarnings("unchecked") + @Override + public T getSettingValue(Settings.Key key) { + return (T) latestSettings.getOrDefault(key, defaultSettings.get(key).getDefault(EMPTY)); + } + + /** + * Register the pair of {key, setting}. + */ + private void register(ImmutableMap.Builder> settingBuilder, + ClusterSettings clusterSettings, Settings.Key key, + Setting setting, + Consumer updater) { + settingBuilder.put(key, setting); + clusterSettings + .addSettingsUpdateConsumer(setting, updater); + } + + /** + * Add the inner class only for UT coverage purpuse. + * Lambda could be much elegant solution. But which is hard to test. + */ + @VisibleForTesting + @RequiredArgsConstructor + class Updater implements Consumer { + private final Settings.Key key; + + @Override + public void accept(Object newValue) { + log.debug("The value of setting [{}] changed to [{}]", key, newValue); + latestSettings.put(key, newValue); + } + } + + /** + * Used by Plugin to init Setting. + */ + public static List> pluginSettings() { + return new ImmutableList.Builder>().add(PPL_QUERY_MEMORY_LIMIT_SETTINGS).build(); + } +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java new file mode 100644 index 0000000000..14a082ee0e --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java @@ -0,0 +1,97 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.mapping.IndexMapping; +import com.amazon.opendistroforelasticsearch.sql.planner.DefaultImplementor; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRelation; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import com.amazon.opendistroforelasticsearch.sql.storage.Table; +import com.google.common.collect.ImmutableMap; +import java.util.HashMap; +import java.util.Map; +import lombok.RequiredArgsConstructor; + +/** Elasticsearch table (index) implementation. */ +@RequiredArgsConstructor +public class ElasticsearchIndex implements Table { + + /** + * Type mapping from Elasticsearch data type to expression type in our type system in query + * engine. TODO: date, geo, ip etc. + */ + private static final Map ES_TYPE_TO_EXPR_TYPE_MAPPING = + ImmutableMap.builder() + .put("text", ExprCoreType.STRING) + .put("keyword", ExprCoreType.STRING) + .put("integer", ExprCoreType.INTEGER) + .put("long", ExprCoreType.LONG) + .put("float", ExprCoreType.FLOAT) + .put("double", ExprCoreType.DOUBLE) + .put("boolean", ExprCoreType.BOOLEAN) + .put("nested", ExprCoreType.ARRAY) + .put("object", ExprCoreType.STRUCT) + .build(); + + /** Elasticsearch client connection. */ + private final ElasticsearchClient client; + + /** Current Elasticsearch index name. */ + private final String indexName; + + /* + * TODO: Assume indexName doesn't have wildcard. + * Need to either handle field name conflicts + * or lazy evaluate when query engine pulls field type. + */ + @Override + public Map getFieldTypes() { + Map fieldTypes = new HashMap<>(); + Map indexMappings = client.getIndexMappings(indexName); + for (IndexMapping indexMapping : indexMappings.values()) { + fieldTypes.putAll(indexMapping.getAllFieldTypes(this::transformESTypeToExprType)); + } + return fieldTypes; + } + + /** TODO: Push down operations to index scan operator as much as possible in future. */ + @Override + public PhysicalPlan implement(LogicalPlan plan) { + ElasticsearchIndexScan indexScan = new ElasticsearchIndexScan(client, indexName); + + /* + * Visit logical plan with index scan as context so logical operators visited, such as + * aggregation, filter, will accumulate (push down) Elasticsearch query and aggregation DSL on + * index scan. + */ + return plan.accept(new DefaultImplementor() { + @Override + public PhysicalPlan visitRelation(LogicalRelation node, ElasticsearchIndexScan context) { + return indexScan; + } + }, + indexScan); + } + + private ExprCoreType transformESTypeToExprType(String esType) { + return ES_TYPE_TO_EXPR_TYPE_MAPPING.getOrDefault(esType, ExprCoreType.UNKNOWN); + } +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexScan.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexScan.java new file mode 100644 index 0000000000..e261782700 --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexScan.java @@ -0,0 +1,82 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.request.ElasticsearchRequest; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.response.ElasticsearchResponse; +import com.amazon.opendistroforelasticsearch.sql.storage.TableScanOperator; +import com.google.common.collect.Iterables; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.elasticsearch.search.SearchHit; + +/** Elasticsearch index scan operator. */ +@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = false) +@ToString(onlyExplicitlyIncluded = true) +public class ElasticsearchIndexScan extends TableScanOperator { + + /** Elasticsearch client. */ + private final ElasticsearchClient client; + + /** Search request. */ + @EqualsAndHashCode.Include @ToString.Include private final ElasticsearchRequest request; + + /** Search response for current batch. */ + private Iterator hits; + + public ElasticsearchIndexScan(ElasticsearchClient client, String indexName) { + this.client = client; + this.request = new ElasticsearchRequest(indexName); + } + + @Override + public void open() { + super.open(); + + // For now pull all results immediately once open + List responses = new ArrayList<>(); + ElasticsearchResponse response = client.search(request); + while (!response.isEmpty()) { + responses.add(response); + response = client.search(request); + } + hits = Iterables.concat(responses.toArray(new ElasticsearchResponse[0])).iterator(); + } + + @Override + public boolean hasNext() { + return hits.hasNext(); + } + + @Override + public ExprValue next() { + return ExprValueUtils.fromObjectValue(hits.next().getSourceAsMap()); + } + + @Override + public void close() { + super.close(); + + client.cleanup(request); + } +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchStorageEngine.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchStorageEngine.java new file mode 100644 index 0000000000..181d55db9c --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchStorageEngine.java @@ -0,0 +1,35 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage; + +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient; +import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; +import com.amazon.opendistroforelasticsearch.sql.storage.Table; +import lombok.RequiredArgsConstructor; + +/** Elasticsearch storage engine implementation. */ +@RequiredArgsConstructor +public class ElasticsearchStorageEngine implements StorageEngine { + + /** Elasticsearch client connection. */ + private final ElasticsearchClient client; + + @Override + public Table getTable(String name) { + return new ElasticsearchIndex(client, name); + } +} diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchNodeClientTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchNodeClientTest.java new file mode 100644 index 0000000000..f9e3e36842 --- /dev/null +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchNodeClientTest.java @@ -0,0 +1,302 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Answers.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.mapping.IndexMapping; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.request.ElasticsearchRequest; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.response.ElasticsearchResponse; +import com.google.common.base.Charsets; +import com.google.common.collect.ImmutableSortedMap; +import com.google.common.io.Resources; +import java.io.IOException; +import java.net.URL; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.lucene.search.TotalHits; +import org.elasticsearch.action.search.ClearScrollRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexAbstraction; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class ElasticsearchNodeClientTest { + + private static final String TEST_MAPPING_FILE = "mappings/accounts.json"; + + @Mock(answer = RETURNS_DEEP_STUBS) + private NodeClient nodeClient; + + @Test + public void getIndexMappings() throws IOException { + URL url = Resources.getResource(TEST_MAPPING_FILE); + String mappings = Resources.toString(url, Charsets.UTF_8); + String indexName = "test"; + ElasticsearchNodeClient client = mockClient(indexName, mappings); + + Map indexMappings = client.getIndexMappings(indexName); + assertEquals(1, indexMappings.size()); + + IndexMapping indexMapping = indexMappings.values().iterator().next(); + assertEquals(20, indexMapping.size()); + assertEquals("text", indexMapping.getFieldType("address")); + assertEquals("integer", indexMapping.getFieldType("age")); + assertEquals("double", indexMapping.getFieldType("balance")); + assertEquals("keyword", indexMapping.getFieldType("city")); + assertEquals("date", indexMapping.getFieldType("birthday")); + assertEquals("geo_point", indexMapping.getFieldType("location")); + assertEquals("some_new_es_type_outside_type_system", indexMapping.getFieldType("new_field")); + assertEquals("text", indexMapping.getFieldType("field with spaces")); + assertEquals("text", indexMapping.getFieldType("employer")); + assertEquals("keyword", indexMapping.getFieldType("employer.raw")); + assertEquals("nested", indexMapping.getFieldType("projects")); + assertEquals("boolean", indexMapping.getFieldType("projects.active")); + assertEquals("date", indexMapping.getFieldType("projects.release")); + assertEquals("nested", indexMapping.getFieldType("projects.members")); + assertEquals("text", indexMapping.getFieldType("projects.members.name")); + assertEquals("object", indexMapping.getFieldType("manager")); + assertEquals("text", indexMapping.getFieldType("manager.name")); + assertEquals("keyword", indexMapping.getFieldType("manager.name.keyword")); + assertEquals("keyword", indexMapping.getFieldType("manager.address")); + assertEquals("long", indexMapping.getFieldType("manager.salary")); + } + + @Test + public void getIndexMappingsWithEmptyMapping() { + String indexName = "test"; + ElasticsearchNodeClient client = mockClient(indexName, ""); + Map indexMappings = client.getIndexMappings(indexName); + assertEquals(1, indexMappings.size()); + + IndexMapping indexMapping = indexMappings.values().iterator().next(); + assertEquals(0, indexMapping.size()); + } + + @Test + public void getIndexMappingsWithIOException() { + String indexName = "test"; + ClusterService clusterService = mockClusterService(indexName, new IOException()); + ElasticsearchNodeClient client = new ElasticsearchNodeClient(clusterService, nodeClient); + + assertThrows(IllegalStateException.class, () -> client.getIndexMappings(indexName)); + } + + @Test + public void getIndexMappingsWithNonExistIndex() { + ElasticsearchNodeClient client = + new ElasticsearchNodeClient(mockClusterService("test"), nodeClient); + + assertThrows(IndexNotFoundException.class, () -> client.getIndexMappings("non_exist_index")); + } + + /** Jacoco enforce this constant lambda be tested. */ + @Test + public void testAllFieldsPredicate() { + assertTrue(ElasticsearchNodeClient.ALL_FIELDS.apply("any_index").test("any_field")); + } + + @Test + public void search() { + ElasticsearchNodeClient client = + new ElasticsearchNodeClient(mock(ClusterService.class), nodeClient); + + // Mock first scroll request + SearchResponse searchResponse = mock(SearchResponse.class); + when(nodeClient.search(any()).actionGet()).thenReturn(searchResponse); + when(searchResponse.getScrollId()).thenReturn("scroll123"); + when(searchResponse.getHits()) + .thenReturn( + new SearchHits( + new SearchHit[] {new SearchHit(1)}, + new TotalHits(1L, TotalHits.Relation.EQUAL_TO), + 1.0F)); + + // Mock second scroll request followed + SearchResponse scrollResponse = mock(SearchResponse.class); + when(nodeClient.searchScroll(any()).actionGet()).thenReturn(scrollResponse); + when(scrollResponse.getScrollId()).thenReturn("scroll456"); + when(scrollResponse.getHits()).thenReturn(SearchHits.empty()); + + // Verify response for first scroll request + ElasticsearchRequest request = new ElasticsearchRequest("test"); + ElasticsearchResponse response1 = client.search(request); + assertFalse(response1.isEmpty()); + + Iterator hits = response1.iterator(); + assertTrue(hits.hasNext()); + assertEquals(new SearchHit(1), hits.next()); + assertFalse(hits.hasNext()); + + // Verify response for second scroll request + ElasticsearchResponse response2 = client.search(request); + assertTrue(response2.isEmpty()); + } + + @Test + void schedule() { + ThreadPool threadPool = mock(ThreadPool.class); + when(nodeClient.threadPool()).thenReturn(threadPool); + + doAnswer( + invocation -> { + Runnable task = invocation.getArgument(0); + task.run(); + return null; + }) + .when(threadPool) + .schedule(any(), any(), any()); + + ElasticsearchNodeClient client = + new ElasticsearchNodeClient(mock(ClusterService.class), nodeClient); + AtomicBoolean isRun = new AtomicBoolean(false); + client.schedule(() -> isRun.set(true)); + assertTrue(isRun.get()); + } + + @Test + void cleanup() { + ClearScrollRequestBuilder requestBuilder = mock(ClearScrollRequestBuilder.class); + when(nodeClient.prepareClearScroll()).thenReturn(requestBuilder); + when(requestBuilder.addScrollId(any())).thenReturn(requestBuilder); + when(requestBuilder.get()).thenReturn(null); + + ElasticsearchNodeClient client = + new ElasticsearchNodeClient(mock(ClusterService.class), nodeClient); + ElasticsearchRequest request = new ElasticsearchRequest("test"); + request.setScrollId("scroll123"); + client.cleanup(request); + assertFalse(request.isScrollStarted()); + + InOrder inOrder = Mockito.inOrder(nodeClient, requestBuilder); + inOrder.verify(nodeClient).prepareClearScroll(); + inOrder.verify(requestBuilder).addScrollId("scroll123"); + inOrder.verify(requestBuilder).get(); + } + + @Test + void cleanupWithoutScrollId() { + ElasticsearchNodeClient client = + new ElasticsearchNodeClient(mock(ClusterService.class), nodeClient); + + ElasticsearchRequest request = new ElasticsearchRequest("test"); + client.cleanup(request); + verify(nodeClient, never()).prepareClearScroll(); + } + + private ElasticsearchNodeClient mockClient(String indexName, String mappings) { + ClusterService clusterService = mockClusterService(indexName, mappings); + return new ElasticsearchNodeClient(clusterService, nodeClient); + } + + /** Mock getAliasAndIndexLookup() only for index name resolve test. */ + public ClusterService mockClusterService(String indexName) { + ClusterService mockService = mock(ClusterService.class); + ClusterState mockState = mock(ClusterState.class); + Metadata mockMetaData = mock(Metadata.class); + + when(mockService.state()).thenReturn(mockState); + when(mockState.metadata()).thenReturn(mockMetaData); + when(mockMetaData.getIndicesLookup()) + .thenReturn(ImmutableSortedMap.of(indexName, mock(IndexAbstraction.class))); + return mockService; + } + + public ClusterService mockClusterService(String indexName, String mappings) { + ClusterService mockService = mock(ClusterService.class); + ClusterState mockState = mock(ClusterState.class); + Metadata mockMetaData = mock(Metadata.class); + + when(mockService.state()).thenReturn(mockState); + when(mockState.metadata()).thenReturn(mockMetaData); + try { + ImmutableOpenMap.Builder> builder = + ImmutableOpenMap.builder(); + ImmutableOpenMap metadata; + if (mappings.isEmpty()) { + metadata = ImmutableOpenMap.of(); + } else { + metadata = IndexMetadata.fromXContent(createParser(mappings)).getMappings(); + } + builder.put(indexName, metadata); + when(mockMetaData.findMappings(any(), any(), any())).thenReturn(builder.build()); + + // IndexNameExpressionResolver use this method to check if index exists. If not, + // IndexNotFoundException is thrown. + when(mockMetaData.getIndicesLookup()) + .thenReturn(ImmutableSortedMap.of(indexName, mock(IndexAbstraction.class))); + } catch (IOException e) { + throw new IllegalStateException("Failed to mock cluster service", e); + } + return mockService; + } + + public ClusterService mockClusterService(String indexName, Throwable t) { + ClusterService mockService = mock(ClusterService.class); + ClusterState mockState = mock(ClusterState.class); + Metadata mockMetaData = mock(Metadata.class); + + when(mockService.state()).thenReturn(mockState); + when(mockState.metadata()).thenReturn(mockMetaData); + try { + when(mockMetaData.findMappings(any(), any(), any())).thenThrow(t); + when(mockMetaData.getIndicesLookup()) + .thenReturn(ImmutableSortedMap.of(indexName, mock(IndexAbstraction.class))); + } catch (IOException e) { + throw new IllegalStateException("Failed to mock cluster service", e); + } + return mockService; + } + + private XContentParser createParser(String mappings) throws IOException { + return XContentType.JSON + .xContent() + .createParser( + NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mappings); + } +} diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchRestClientTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchRestClientTest.java new file mode 100644 index 0000000000..21b8da5acc --- /dev/null +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchRestClientTest.java @@ -0,0 +1,207 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Answers.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.mapping.IndexMapping; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.request.ElasticsearchRequest; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.response.ElasticsearchResponse; +import com.google.common.base.Charsets; +import com.google.common.collect.ImmutableMap; +import com.google.common.io.Resources; +import java.io.IOException; +import java.net.URL; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.lucene.search.TotalHits; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.indices.GetMappingsRequest; +import org.elasticsearch.client.indices.GetMappingsResponse; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class ElasticsearchRestClientTest { + + private static final String TEST_MAPPING_FILE = "mappings/accounts.json"; + + @Mock(answer = RETURNS_DEEP_STUBS) + private RestHighLevelClient restClient; + + private ElasticsearchRestClient client; + + @BeforeEach + void setUp() { + client = new ElasticsearchRestClient(restClient); + } + + @Test + void getIndexMappings() throws IOException { + URL url = Resources.getResource(TEST_MAPPING_FILE); + String mappings = Resources.toString(url, Charsets.UTF_8); + String indexName = "test"; + + GetMappingsResponse response = mock(GetMappingsResponse.class); + when(response.mappings()).thenReturn(mockFieldMappings(indexName, mappings)); + when(restClient.indices().getMapping(any(GetMappingsRequest.class), any())) + .thenReturn(response); + + Map indexMappings = client.getIndexMappings(indexName); + assertEquals(1, indexMappings.size()); + + IndexMapping indexMapping = indexMappings.values().iterator().next(); + assertEquals(20, indexMapping.size()); + assertEquals("text", indexMapping.getFieldType("address")); + assertEquals("integer", indexMapping.getFieldType("age")); + assertEquals("double", indexMapping.getFieldType("balance")); + assertEquals("keyword", indexMapping.getFieldType("city")); + assertEquals("date", indexMapping.getFieldType("birthday")); + assertEquals("geo_point", indexMapping.getFieldType("location")); + assertEquals("some_new_es_type_outside_type_system", indexMapping.getFieldType("new_field")); + assertEquals("text", indexMapping.getFieldType("field with spaces")); + assertEquals("text", indexMapping.getFieldType("employer")); + assertEquals("keyword", indexMapping.getFieldType("employer.raw")); + assertEquals("nested", indexMapping.getFieldType("projects")); + assertEquals("boolean", indexMapping.getFieldType("projects.active")); + assertEquals("date", indexMapping.getFieldType("projects.release")); + assertEquals("nested", indexMapping.getFieldType("projects.members")); + assertEquals("text", indexMapping.getFieldType("projects.members.name")); + assertEquals("object", indexMapping.getFieldType("manager")); + assertEquals("text", indexMapping.getFieldType("manager.name")); + assertEquals("keyword", indexMapping.getFieldType("manager.name.keyword")); + assertEquals("keyword", indexMapping.getFieldType("manager.address")); + assertEquals("long", indexMapping.getFieldType("manager.salary")); + } + + @Test + void getIndexMappingsWithIOException() throws IOException { + when(restClient.indices().getMapping(any(GetMappingsRequest.class), any())) + .thenThrow(new IOException()); + assertThrows(IllegalStateException.class, () -> client.getIndexMappings("test")); + } + + @Test + void search() throws IOException { + // Mock first scroll request + SearchResponse searchResponse = mock(SearchResponse.class); + when(restClient.search(any(), any())).thenReturn(searchResponse); + when(searchResponse.getScrollId()).thenReturn("scroll123"); + when(searchResponse.getHits()) + .thenReturn( + new SearchHits( + new SearchHit[] {new SearchHit(1)}, + new TotalHits(1L, TotalHits.Relation.EQUAL_TO), + 1.0F)); + + // Mock second scroll request followed + SearchResponse scrollResponse = mock(SearchResponse.class); + when(restClient.scroll(any(), any())).thenReturn(scrollResponse); + when(scrollResponse.getScrollId()).thenReturn("scroll456"); + when(scrollResponse.getHits()).thenReturn(SearchHits.empty()); + + // Verify response for first scroll request + ElasticsearchRequest request = new ElasticsearchRequest("test"); + ElasticsearchResponse response1 = client.search(request); + assertFalse(response1.isEmpty()); + + Iterator hits = response1.iterator(); + assertTrue(hits.hasNext()); + assertEquals(new SearchHit(1), hits.next()); + assertFalse(hits.hasNext()); + + // Verify response for second scroll request + ElasticsearchResponse response2 = client.search(request); + assertTrue(response2.isEmpty()); + } + + @Test + void searchWithIOException() throws IOException { + when(restClient.search(any(), any())).thenThrow(new IOException()); + assertThrows( + IllegalStateException.class, () -> client.search(new ElasticsearchRequest("test"))); + } + + @Test + void schedule() { + AtomicBoolean isRun = new AtomicBoolean(false); + client.schedule( + () -> { + isRun.set(true); + }); + assertTrue(isRun.get()); + } + + @Test + void cleanup() throws IOException { + ElasticsearchRequest request = new ElasticsearchRequest("test"); + request.setScrollId("scroll123"); + client.cleanup(request); + verify(restClient).clearScroll(any(), any()); + assertFalse(request.isScrollStarted()); + } + + @Test + void cleanupWithoutScrollId() throws IOException { + ElasticsearchRequest request = new ElasticsearchRequest("test"); + client.cleanup(request); + verify(restClient, never()).clearScroll(any(), any()); + } + + @Test + void cleanupWithIOException() throws IOException { + when(restClient.clearScroll(any(), any())).thenThrow(new IOException()); + + ElasticsearchRequest request = new ElasticsearchRequest("test"); + request.setScrollId("scroll123"); + assertThrows(IllegalStateException.class, () -> client.cleanup(request)); + } + + private Map mockFieldMappings(String indexName, String mappings) + throws IOException { + return ImmutableMap.of(indexName, IndexMetadata.fromXContent(createParser(mappings)).mapping()); + } + + private XContentParser createParser(String mappings) throws IOException { + return XContentType.JSON + .xContent() + .createParser( + NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mappings); + } +} diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/ElasticsearchExecutionEngineTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/ElasticsearchExecutionEngineTest.java new file mode 100644 index 0000000000..f1ca7ac1f8 --- /dev/null +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/ElasticsearchExecutionEngineTest.java @@ -0,0 +1,152 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.tupleValue; +import static com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.QueryResponse; +import static com.google.common.collect.ImmutableMap.of; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.common.response.ResponseListener; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor.protector.ElasticsearchExecutionProtector; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import com.amazon.opendistroforelasticsearch.sql.storage.TableScanOperator; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import lombok.RequiredArgsConstructor; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class ElasticsearchExecutionEngineTest { + + @Mock private ElasticsearchClient client; + + @Mock private ElasticsearchExecutionProtector protector; + + @BeforeEach + void setUp() { + doAnswer( + invocation -> { + // Run task immediately + Runnable task = invocation.getArgument(0); + task.run(); + return null; + }) + .when(client) + .schedule(any()); + } + + @Test + void executeSuccessfully() { + List expected = + Arrays.asList( + tupleValue(of("name", "John", "age", 20)), tupleValue(of("name", "Allen", "age", 30))); + FakePhysicalPlan plan = new FakePhysicalPlan(expected.iterator()); + when(protector.protect(plan)).thenReturn(plan); + + ElasticsearchExecutionEngine executor = new ElasticsearchExecutionEngine(client, protector); + List actual = new ArrayList<>(); + executor.execute( + plan, + new ResponseListener() { + @Override + public void onResponse(QueryResponse response) { + actual.addAll(response.getResults()); + } + + @Override + public void onFailure(Exception e) { + fail("Error occurred during execution", e); + } + }); + + assertTrue(plan.hasOpen); + assertEquals(expected, actual); + assertTrue(plan.hasClosed); + } + + @Test + void executeWithFailure() { + PhysicalPlan plan = mock(PhysicalPlan.class); + RuntimeException expected = new RuntimeException("Execution error"); + when(plan.hasNext()).thenThrow(expected); + when(protector.protect(plan)).thenReturn(plan); + + ElasticsearchExecutionEngine executor = new ElasticsearchExecutionEngine(client, protector); + AtomicReference actual = new AtomicReference<>(); + executor.execute( + plan, + new ResponseListener() { + @Override + public void onResponse(QueryResponse response) { + fail("Expected error didn't happen"); + } + + @Override + public void onFailure(Exception e) { + actual.set(e); + } + }); + assertEquals(expected, actual.get()); + verify(plan).close(); + } + + @RequiredArgsConstructor + private static class FakePhysicalPlan extends TableScanOperator { + private final Iterator it; + private boolean hasOpen; + private boolean hasClosed; + + @Override + public void open() { + super.open(); + hasOpen = true; + } + + @Override + public void close() { + super.close(); + hasClosed = true; + } + + @Override + public boolean hasNext() { + return it.hasNext(); + } + + @Override + public ExprValue next() { + return it.next(); + } + } +} diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/ElasticsearchExecutionProtectorTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/ElasticsearchExecutionProtectorTest.java new file mode 100644 index 0000000000..433d7592c1 --- /dev/null +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/ElasticsearchExecutionProtectorTest.java @@ -0,0 +1,148 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.literal; +import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.ref; +import static com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlanDSL.filter; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor.protector.ElasticsearchExecutionProtector; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor.protector.ResourceMonitorPlan; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.ElasticsearchIndexScan; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; +import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.AvgAggregator; +import com.amazon.opendistroforelasticsearch.sql.monitor.ResourceMonitor; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlanDSL; +import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class ElasticsearchExecutionProtectorTest { + @Mock + private ElasticsearchClient client; + + @Mock + private ResourceMonitor resourceMonitor; + + private ElasticsearchExecutionProtector executionProtector; + + @BeforeEach + public void setup() { + executionProtector = new ElasticsearchExecutionProtector(resourceMonitor); + } + + @Test + public void testProtectIndexScan() { + String indexName = "test"; + ReferenceExpression include = ref("age", INTEGER); + ReferenceExpression exclude = ref("name", STRING); + ReferenceExpression dedupeField = ref("name", STRING); + Expression filterExpr = literal(ExprBooleanValue.ofTrue()); + List groupByExprs = Arrays.asList(ref("age", INTEGER)); + List aggregators = Arrays.asList(new AvgAggregator(groupByExprs, DOUBLE)); + Map mappings = + ImmutableMap.of(ref("name", STRING), ref("lastname", STRING)); + Pair newEvalField = + ImmutablePair.of(ref("name1", STRING), ref("name", STRING)); + Integer sortCount = 100; + Pair sortField = + ImmutablePair.of(Sort.SortOption.PPL_ASC, ref("name1", STRING)); + + assertEquals( + PhysicalPlanDSL.project( + PhysicalPlanDSL.dedupe( + PhysicalPlanDSL.sort( + PhysicalPlanDSL.eval( + PhysicalPlanDSL.remove( + PhysicalPlanDSL.rename( + PhysicalPlanDSL.agg( + filter( + resourceMonitor(new ElasticsearchIndexScan(client, + indexName)), + filterExpr), + aggregators, + groupByExprs), + mappings), + exclude), + newEvalField), + sortCount, + sortField), + dedupeField), + include), + executionProtector.protect( + PhysicalPlanDSL.project( + PhysicalPlanDSL.dedupe( + PhysicalPlanDSL.sort( + PhysicalPlanDSL.eval( + PhysicalPlanDSL.remove( + PhysicalPlanDSL.rename( + PhysicalPlanDSL.agg( + filter( + new ElasticsearchIndexScan(client, indexName), + filterExpr), + aggregators, + groupByExprs), + mappings), + exclude), + newEvalField), + sortCount, + sortField), + dedupeField), + include)) + ); + } + + @Test + public void testWithoutProtection() { + Expression filterExpr = literal(ExprBooleanValue.ofTrue()); + + assertEquals( + filter( + filter(null, filterExpr), + filterExpr), + executionProtector.protect( + filter( + filter(null, filterExpr), + filterExpr) + ) + ); + } + + PhysicalPlan resourceMonitor(PhysicalPlan input) { + return new ResourceMonitorPlan(input, resourceMonitor); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/ResourceMonitorPlanTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/ResourceMonitorPlanTest.java new file mode 100644 index 0000000000..de2e0d157b --- /dev/null +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/ResourceMonitorPlanTest.java @@ -0,0 +1,103 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor.protector.ResourceMonitorPlan; +import com.amazon.opendistroforelasticsearch.sql.monitor.ResourceMonitor; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlanNodeVisitor; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class ResourceMonitorPlanTest { + @Mock + private PhysicalPlan plan; + + @Mock + private ResourceMonitor resourceMonitor; + + @Mock + private PhysicalPlanNodeVisitor visitor; + + @Mock + private Object context; + + private ResourceMonitorPlan monitorPlan; + + @BeforeEach + public void setup() { + monitorPlan = new ResourceMonitorPlan(plan, resourceMonitor); + } + + @Test + void openExceedResourceLimit() { + when(resourceMonitor.isHealthy()).thenReturn(false); + + IllegalStateException exception = + assertThrows(IllegalStateException.class, () -> monitorPlan.open()); + assertEquals("resource is not enough to run the query, quit.", exception.getMessage()); + } + + @Test + void openSuccess() { + when(resourceMonitor.isHealthy()).thenReturn(true); + + monitorPlan.open(); + verify(plan, times(1)).open(); + } + + @Test + void nextSuccess() { + monitorPlan.next(); + verify(plan, times(1)).next(); + } + + @Test + void hasNextSuccess() { + monitorPlan.hasNext(); + verify(plan, times(1)).hasNext(); + } + + @Test + void closeSuccess() { + monitorPlan.close(); + verify(plan, times(1)).close(); + } + + @Test + void getChildSuccess() { + monitorPlan.getChild(); + verify(plan, times(1)).getChild(); + } + + @Test + void acceptSuccess() { + monitorPlan.accept(visitor, context); + verify(plan, times(1)).accept(visitor, context); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/protector/NoopExecutionProtectorTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/protector/NoopExecutionProtectorTest.java new file mode 100644 index 0000000000..e36baf35cf --- /dev/null +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/protector/NoopExecutionProtectorTest.java @@ -0,0 +1,41 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor.protector; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class NoopExecutionProtectorTest { + + @Mock + private PhysicalPlan plan; + + @Test + void protect() { + NoopExecutionProtector executionProtector = new NoopExecutionProtector(); + PhysicalPlan protectedPlan = executionProtector.protect(plan); + + assertEquals(plan, protectedPlan); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/mapping/IndexMappingTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/mapping/IndexMappingTest.java new file mode 100644 index 0000000000..9dee52341f --- /dev/null +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/mapping/IndexMappingTest.java @@ -0,0 +1,47 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.mapping; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.hasEntry; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class IndexMappingTest { + + @Test + public void getFieldType() { + IndexMapping indexMapping = new IndexMapping(ImmutableMap.of("name", "text")); + assertEquals("text", indexMapping.getFieldType("name")); + assertNull(indexMapping.getFieldType("not_exist")); + } + + @Test + public void getAllFieldTypes() { + IndexMapping indexMapping = new IndexMapping(ImmutableMap.of("name", "text", "age", "int")); + Map fieldTypes = indexMapping.getAllFieldTypes(type -> "our_type"); + assertThat( + fieldTypes, + allOf(aMapWithSize(2), hasEntry("name", "our_type"), hasEntry("age", "our_type"))); + } +} diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/monitor/ElasticsearchMemoryHealthyTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/monitor/ElasticsearchMemoryHealthyTest.java new file mode 100644 index 0000000000..08aeedf68d --- /dev/null +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/monitor/ElasticsearchMemoryHealthyTest.java @@ -0,0 +1,88 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.monitor; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class ElasticsearchMemoryHealthyTest { + + @Mock + private ElasticsearchMemoryHealthy.RandomFail randomFail; + + @Mock + private ElasticsearchMemoryHealthy.MemoryUsage memoryUsage; + + private ElasticsearchMemoryHealthy monitor; + + @BeforeEach + public void setup() { + monitor = new ElasticsearchMemoryHealthy(randomFail, memoryUsage); + } + + @Test + void isMemoryHealthy() { + when(memoryUsage.usage()).thenReturn(10L); + + assertTrue(monitor.isMemoryHealthy(11L)); + } + + @Test + void memoryUsageExceedLimitFastFailure() { + when(memoryUsage.usage()).thenReturn(10L); + when(randomFail.shouldFail()).thenReturn(true); + + assertThrows(ElasticsearchMemoryHealthy.MemoryUsageExceedFastFailureException.class, + () -> monitor.isMemoryHealthy(9L)); + } + + @Test + void memoryUsageExceedLimitWithoutFastFailure() { + when(memoryUsage.usage()).thenReturn(10L); + when(randomFail.shouldFail()).thenReturn(false); + + assertThrows(ElasticsearchMemoryHealthy.MemoryUsageExceedException.class, + () -> monitor.isMemoryHealthy(9L)); + } + + @Test + void constructElasticsearchMemoryMonitorWithoutArguments() { + ElasticsearchMemoryHealthy monitor = new ElasticsearchMemoryHealthy(); + assertNotNull(monitor); + } + + @Test + void randomFail() { + ElasticsearchMemoryHealthy.RandomFail randomFail = new ElasticsearchMemoryHealthy.RandomFail(); + assertNotNull(randomFail.shouldFail()); + } + + @Test + void setMemoryUsage() { + ElasticsearchMemoryHealthy.MemoryUsage usage = + new ElasticsearchMemoryHealthy.MemoryUsage(); + assertTrue(usage.usage() > 0); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/monitor/ElasticsearchResourceMonitorTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/monitor/ElasticsearchResourceMonitorTest.java new file mode 100644 index 0000000000..c8a58f3422 --- /dev/null +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/monitor/ElasticsearchResourceMonitorTest.java @@ -0,0 +1,90 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.monitor; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.common.setting.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class ElasticsearchResourceMonitorTest { + + @Mock + private Settings settings; + + @Mock + private ElasticsearchMemoryHealthy memoryMonitor; + + @BeforeEach + public void setup() { + when(settings.getSettingValue(Settings.Key.PPL_QUERY_MEMORY_LIMIT)) + .thenReturn(new ByteSizeValue(10L)); + } + + @Test + void isHealthy() { + when(memoryMonitor.isMemoryHealthy(anyLong())).thenReturn(true); + + ElasticsearchResourceMonitor resourceMonitor = + new ElasticsearchResourceMonitor(settings, memoryMonitor); + assertTrue(resourceMonitor.isHealthy()); + } + + @Test + void notHealthyFastFailure() { + when(memoryMonitor.isMemoryHealthy(anyLong())).thenThrow( + ElasticsearchMemoryHealthy.MemoryUsageExceedFastFailureException.class); + + ElasticsearchResourceMonitor resourceMonitor = + new ElasticsearchResourceMonitor(settings, memoryMonitor); + assertFalse(resourceMonitor.isHealthy()); + verify(memoryMonitor, times(1)).isMemoryHealthy(anyLong()); + } + + @Test + void notHealthyWithRetry() { + when(memoryMonitor.isMemoryHealthy(anyLong())).thenThrow( + ElasticsearchMemoryHealthy.MemoryUsageExceedException.class); + + ElasticsearchResourceMonitor resourceMonitor = + new ElasticsearchResourceMonitor(settings, memoryMonitor); + assertFalse(resourceMonitor.isHealthy()); + verify(memoryMonitor, times(3)).isMemoryHealthy(anyLong()); + } + + @Test + void healthyWithRetry() { + + when(memoryMonitor.isMemoryHealthy(anyLong())).thenThrow( + ElasticsearchMemoryHealthy.MemoryUsageExceedException.class).thenReturn(true); + + ElasticsearchResourceMonitor resourceMonitor = + new ElasticsearchResourceMonitor(settings, memoryMonitor); + assertTrue(resourceMonitor.isHealthy()); + verify(memoryMonitor, times(2)).isMemoryHealthy(anyLong()); + } +} diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/ElasticsearchRequestTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/ElasticsearchRequestTest.java new file mode 100644 index 0000000000..cd6600d100 --- /dev/null +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/request/ElasticsearchRequestTest.java @@ -0,0 +1,62 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.request; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchScrollRequest; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.junit.jupiter.api.Test; + +class ElasticsearchRequestTest { + + private final ElasticsearchRequest request = new ElasticsearchRequest("test"); + + @Test + void searchRequest() { + request.getSourceBuilder().query(QueryBuilders.termQuery("name", "John")); + + assertEquals( + new SearchRequest() + .indices("test") + .scroll(ElasticsearchRequest.DEFAULT_SCROLL_TIMEOUT) + .source(new SearchSourceBuilder().query(QueryBuilders.termQuery("name", "John"))), + request.searchRequest()); + } + + @Test + void isScrollStarted() { + assertFalse(request.isScrollStarted()); + + request.setScrollId("scroll123"); + assertTrue(request.isScrollStarted()); + } + + @Test + void scrollRequest() { + request.setScrollId("scroll123"); + assertEquals( + new SearchScrollRequest() + .scroll(ElasticsearchRequest.DEFAULT_SCROLL_TIMEOUT) + .scrollId("scroll123"), + request.scrollRequest()); + } +} diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/response/ElasticsearchResponseTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/response/ElasticsearchResponseTest.java new file mode 100644 index 0000000000..68ea1af73d --- /dev/null +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/response/ElasticsearchResponseTest.java @@ -0,0 +1,79 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.response; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.when; + +import org.apache.lucene.search.TotalHits; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class ElasticsearchResponseTest { + + @Mock private SearchResponse esResponse; + + @BeforeEach + void setUp() { + when(esResponse.getHits()) + .thenReturn( + new SearchHits( + new SearchHit[] {new SearchHit(1), new SearchHit(2)}, + new TotalHits(2L, TotalHits.Relation.EQUAL_TO), + 1.0F)); + } + + @Test + void isEmpty() { + ElasticsearchResponse response1 = new ElasticsearchResponse(esResponse); + assertFalse(response1.isEmpty()); + + when(esResponse.getHits()).thenReturn(SearchHits.empty()); + ElasticsearchResponse response2 = new ElasticsearchResponse(esResponse); + assertTrue(response2.isEmpty()); + + when(esResponse.getHits()) + .thenReturn(new SearchHits(null, new TotalHits(0, TotalHits.Relation.EQUAL_TO), 0)); + ElasticsearchResponse response3 = new ElasticsearchResponse(esResponse); + assertTrue(response3.isEmpty()); + } + + @Test + void iterator() { + int i = 0; + for (SearchHit hit : new ElasticsearchResponse(esResponse)) { + if (i == 0) { + assertEquals(new SearchHit(1), hit); + } else if (i == 1) { + assertEquals(new SearchHit(2), hit); + } else { + fail("More search hits returned than expected"); + } + i++; + } + } +} diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/setting/ElasticsearchSettingsTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/setting/ElasticsearchSettingsTest.java new file mode 100644 index 0000000000..a50c2a2ed8 --- /dev/null +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/setting/ElasticsearchSettingsTest.java @@ -0,0 +1,67 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.setting; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import com.amazon.opendistroforelasticsearch.sql.common.setting.Settings; +import java.util.List; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class ElasticsearchSettingsTest { + + @Mock + private ClusterSettings clusterSettings; + + @Test + void getSettingValue() { + ElasticsearchSettings settings = new ElasticsearchSettings(clusterSettings); + ByteSizeValue sizeValue = settings.getSettingValue(Settings.Key.PPL_QUERY_MEMORY_LIMIT); + + assertNotNull(sizeValue); + } + + @Test + void pluginSettings() { + List> settings = ElasticsearchSettings.pluginSettings(); + + assertFalse(settings.isEmpty()); + } + + @Test + void update() { + ElasticsearchSettings settings = new ElasticsearchSettings(clusterSettings); + ByteSizeValue oldValue = settings.getSettingValue(Settings.Key.PPL_QUERY_MEMORY_LIMIT); + ElasticsearchSettings.Updater updater = + settings.new Updater(Settings.Key.PPL_QUERY_MEMORY_LIMIT); + updater.accept(new ByteSizeValue(0L)); + + ByteSizeValue newValue = settings.getSettingValue(Settings.Key.PPL_QUERY_MEMORY_LIMIT); + + assertNotEquals(newValue.getBytes(), oldValue.getBytes()); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexScanTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexScanTest.java new file mode 100644 index 0000000000..12e8caebd3 --- /dev/null +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexScanTest.java @@ -0,0 +1,116 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.response.ElasticsearchResponse; +import java.util.Arrays; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.search.SearchHit; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.stubbing.Answer; + +@ExtendWith(MockitoExtension.class) +class ElasticsearchIndexScanTest { + + @Mock private ElasticsearchClient client; + + @Test + void queryEmptyResult() { + mockResponse(); + try (ElasticsearchIndexScan indexScan = new ElasticsearchIndexScan(client, "test")) { + indexScan.open(); + assertFalse(indexScan.hasNext()); + } + verify(client).cleanup(any()); + } + + @Test + void queryAllResults() { + mockResponse( + new SearchHit[] {employee(1, "John", "IT"), employee(2, "Smith", "HR")}, + new SearchHit[] {employee(3, "Allen", "IT")}); + + try (ElasticsearchIndexScan indexScan = new ElasticsearchIndexScan(client, "employees")) { + indexScan.open(); + + assertTrue(indexScan.hasNext()); + assertEquals(tupleValue(employee(1, "John", "IT")), indexScan.next()); + + assertTrue(indexScan.hasNext()); + assertEquals(tupleValue(employee(2, "Smith", "HR")), indexScan.next()); + + assertTrue(indexScan.hasNext()); + assertEquals(tupleValue(employee(3, "Allen", "IT")), indexScan.next()); + + assertFalse(indexScan.hasNext()); + } + verify(client).cleanup(any()); + } + + private void mockResponse(SearchHit[]... searchHitBatches) { + when(client.search(any())) + .thenAnswer( + new Answer() { + private int batchNum; + + @Override + public ElasticsearchResponse answer(InvocationOnMock invocation) { + ElasticsearchResponse response = mock(ElasticsearchResponse.class); + int totalBatch = searchHitBatches.length; + if (batchNum < totalBatch) { + when(response.isEmpty()).thenReturn(false); + SearchHit[] searchHit = searchHitBatches[batchNum]; + when(response.iterator()).thenReturn(Arrays.asList(searchHit).iterator()); + } else if (batchNum == totalBatch) { + when(response.isEmpty()).thenReturn(true); + } else { + fail("Search request after empty response returned already"); + } + + batchNum++; + return response; + } + }); + } + + protected SearchHit employee(int docId, String name, String department) { + SearchHit hit = new SearchHit(docId); + hit.sourceRef( + new BytesArray("{\"name\":\"" + name + "\",\"department\":\"" + department + "\"}")); + return hit; + } + + private ExprValue tupleValue(SearchHit hit) { + return ExprValueUtils.tupleValue(hit.getSourceAsMap()); + } +} diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexTest.java new file mode 100644 index 0000000000..63ebd28a1c --- /dev/null +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexTest.java @@ -0,0 +1,174 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.literal; +import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.ref; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.aggregation; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.eval; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.filter; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.project; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.relation; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.remove; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.rename; +import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.sort; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.hasEntry; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort.SortOption; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.mapping.IndexMapping; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; +import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.AvgAggregator; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlanDSL; +import com.amazon.opendistroforelasticsearch.sql.storage.Table; +import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class ElasticsearchIndexTest { + + @Mock private ElasticsearchClient client; + + @Test + void getFieldTypes() { + when(client.getIndexMappings("test")) + .thenReturn( + ImmutableMap.of( + "test", + new IndexMapping( + ImmutableMap.builder() + .put("name", "keyword") + .put("address", "text") + .put("age", "integer") + .put("account_number", "long") + .put("balance1", "float") + .put("balance2", "double") + .put("gender", "boolean") + .put("family", "nested") + .put("employer", "object") + .put("birthday", "date") + .build()))); + + Table index = new ElasticsearchIndex(client, "test"); + Map fieldTypes = index.getFieldTypes(); + assertThat( + fieldTypes, + allOf( + aMapWithSize(10), + hasEntry("name", ExprCoreType.STRING), + hasEntry("address", ExprCoreType.STRING), + hasEntry("age", ExprCoreType.INTEGER), + hasEntry("account_number", ExprCoreType.LONG), + hasEntry("balance1", ExprCoreType.FLOAT), + hasEntry("balance2", ExprCoreType.DOUBLE), + hasEntry("gender", ExprCoreType.BOOLEAN), + hasEntry("family", ExprCoreType.ARRAY), + hasEntry("employer", ExprCoreType.STRUCT), + hasEntry("birthday", ExprCoreType.UNKNOWN))); + } + + @Test + void implementRelationOperatorOnly() { + String indexName = "test"; + LogicalPlan plan = relation(indexName); + Table index = new ElasticsearchIndex(client, indexName); + assertEquals(new ElasticsearchIndexScan(client, indexName), index.implement(plan)); + } + + @Test + void implementOtherLogicalOperators() { + String indexName = "test"; + ReferenceExpression include = ref("age", INTEGER); + ReferenceExpression exclude = ref("name", STRING); + ReferenceExpression dedupeField = ref("name", STRING); + Expression filterExpr = literal(ExprBooleanValue.ofTrue()); + List groupByExprs = Arrays.asList(ref("age", INTEGER)); + List aggregators = Arrays.asList(new AvgAggregator(groupByExprs, DOUBLE)); + Map mappings = + ImmutableMap.of(ref("name", STRING), ref("lastname", STRING)); + Pair newEvalField = + ImmutablePair.of(ref("name1", STRING), ref("name", STRING)); + Integer sortCount = 100; + Pair sortField = + ImmutablePair.of(Sort.SortOption.PPL_ASC, ref("name1", STRING)); + + LogicalPlan plan = + project( + LogicalPlanDSL.dedupe( + sort( + eval( + remove( + rename( + aggregation( + filter(relation(indexName), filterExpr), + aggregators, + groupByExprs), + mappings), + exclude), + newEvalField), + sortCount, + sortField), + dedupeField), + include); + + Table index = new ElasticsearchIndex(client, indexName); + assertEquals( + PhysicalPlanDSL.project( + PhysicalPlanDSL.dedupe( + PhysicalPlanDSL.sort( + PhysicalPlanDSL.eval( + PhysicalPlanDSL.remove( + PhysicalPlanDSL.rename( + PhysicalPlanDSL.agg( + PhysicalPlanDSL.filter( + new ElasticsearchIndexScan(client, indexName), filterExpr), + aggregators, + groupByExprs), + mappings), + exclude), + newEvalField), + sortCount, + sortField), + dedupeField), + include), + index.implement(plan)); + } +} diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchStorageEngineTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchStorageEngineTest.java new file mode 100644 index 0000000000..f17455f988 --- /dev/null +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchStorageEngineTest.java @@ -0,0 +1,39 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient; +import com.amazon.opendistroforelasticsearch.sql.storage.Table; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class ElasticsearchStorageEngineTest { + + @Mock private ElasticsearchClient client; + + @Test + public void getTable() { + ElasticsearchStorageEngine engine = new ElasticsearchStorageEngine(client); + Table table = engine.getTable("test"); + assertNotNull(table); + } +} diff --git a/elasticsearch/src/test/resources/mappings/accounts.json b/elasticsearch/src/test/resources/mappings/accounts.json new file mode 100644 index 0000000000..a7024b6490 --- /dev/null +++ b/elasticsearch/src/test/resources/mappings/accounts.json @@ -0,0 +1,92 @@ +{ + "accounts": { + "mappings": { + "_doc": { + "properties": { + "address": { + "type": "text" + }, + "age": { + "type": "integer" + }, + "balance": { + "type": "double" + }, + "city": { + "type": "keyword" + }, + "birthday": { + "type": "date" + }, + "location": { + "type": "geo_point" + }, + "new_field": { + "type": "some_new_es_type_outside_type_system" + }, + "field with spaces": { + "type": "text" + }, + "employer": { + "type": "text", + "fields": { + "raw": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "projects": { + "type": "nested", + "properties": { + "members": { + "type": "nested", + "properties": { + "name": { + "type": "text" + } + } + }, + "active": { + "type": "boolean" + }, + "release": { + "type": "date" + } + } + }, + "manager": { + "properties": { + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "address": { + "type": "keyword" + }, + "salary": { + "type": "long" + } + } + } + } + } + }, + "settings": { + "index": { + "number_of_shards": 5, + "number_of_replicas": 0, + "version": { + "created": "6050399" + } + } + }, + "mapping_version": "1", + "settings_version": "1" + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/elasticsearch/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 0000000000..ca6ee9cea8 --- /dev/null +++ b/elasticsearch/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index c740b5d54c..5896cf54df 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,4 +13,4 @@ # permissions and limitations under the License. # -version=1.8.0 +version=1.9.0 diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index cc3d06cf2c..75afb6ed35 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -13,7 +13,7 @@ # permissions and limitations under the License. # -distributionUrl=https\://services.gradle.org/distributions/gradle-6.4-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.5-all.zip distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStorePath=wrapper/dists diff --git a/integ-test/build.gradle b/integ-test/build.gradle new file mode 100644 index 0000000000..3719344ff0 --- /dev/null +++ b/integ-test/build.gradle @@ -0,0 +1,149 @@ +import org.elasticsearch.gradle.test.RestIntegTestTask + +apply plugin: 'elasticsearch.build' +apply plugin: 'elasticsearch.rest-test' +apply plugin: 'java' +apply plugin: 'io.freefair.lombok' + +ext { + projectSubstitutions = [:] + licenseFile = rootProject.file('LICENSE.TXT') + noticeFile = rootProject.file('NOTICE') +} + +tasks.withType(licenseHeaders.class) { + additionalLicense 'AL ', 'Apache', 'Licensed under the Apache License, Version 2.0 (the "License")' +} + +validateNebulaPom.enabled = false + +repositories { + mavenCentral() +} + +configurations.all { + exclude group: "commons-logging", module: "commons-logging" +} + +dependencies { + testCompile group: 'org.elasticsearch.test', name: 'framework', version: "${es_version}" + testCompile group: 'org.elasticsearch.client', name: 'elasticsearch-rest-high-level-client', version: "${es_version}" + testCompile group: 'org.elasticsearch.client', name: 'elasticsearch-rest-client', version: "${es_version}" + testCompile group: 'org.hamcrest', name: 'hamcrest', version: '2.1' + testCompile group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.11.1' + testCompile project(':plugin') + testCompile project(':legacy') + testImplementation('org.junit.jupiter:junit-jupiter-api:5.6.2') + testRuntimeOnly('org.junit.jupiter:junit-jupiter-engine:5.6.2') + + // JDBC drivers for comparison test. Somehow Apache Derby throws security permission exception. + testCompile group: 'com.amazon.opendistroforelasticsearch.client', name: 'opendistro-sql-jdbc', version: '1.8.0.0' + testCompile group: 'com.h2database', name: 'h2', version: '1.4.200' + testCompile group: 'org.xerial', name: 'sqlite-jdbc', version: '3.28.0' + //testCompile group: 'org.apache.derby', name: 'derby', version: '10.15.1.3' +} + +dependencyLicenses.enabled = false +testingConventions.enabled = false +checkstyleTest.ignoreFailures = true +forbiddenApisTest.enabled = false + +compileTestJava { + options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) + doFirst { // Necessary because of many warnings in legacy SQL IT + options.compilerArgs.remove('-Werror') + options.compilerArgs.remove('-Xdoclint:all') + } +} + +tasks.integTest.dependsOn(':plugin:bundlePlugin') +testClusters.integTest { + testDistribution = 'oss' + plugin file(tasks.getByPath(':plugin:bundlePlugin').archiveFile) +} + +integTest.runner { + systemProperty 'tests.security.manager', 'false' + systemProperty('project.root', project.projectDir.absolutePath) + + systemProperty "https", System.getProperty("https") + systemProperty "user", System.getProperty("user") + systemProperty "password", System.getProperty("password") + + // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for + // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time. + doFirst { systemProperty 'cluster.debug', getDebug()} + + if (System.getProperty("test.debug") != null) { + jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' + } + + include 'com/amazon/opendistroforelasticsearch/sql/ppl/**/*IT.class' + include 'com/amazon/opendistroforelasticsearch/sql/legacy/**/*IT.class' + exclude 'com/amazon/opendistroforelasticsearch/sql/doctest/**/*IT.class' + exclude 'com/amazon/opendistroforelasticsearch/sql/correctness/**' +} + + +task docTest(type: RestIntegTestTask) { + dependsOn ':plugin:bundlePlugin' + runner { + systemProperty 'tests.security.manager', 'false' + systemProperty('project.root', project.projectDir.absolutePath) + + // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for + // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time. + doFirst { systemProperty 'cluster.debug', getDebug()} + + if (System.getProperty("test.debug") != null) { + jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' + } + + include 'com/amazon/opendistroforelasticsearch/sql/doctest/**/*IT.class' + exclude 'com/amazon/opendistroforelasticsearch/sql/correctness/**/*IT.class' + exclude 'com/amazon/opendistroforelasticsearch/sql/ppl/**/*IT.class' + exclude 'com/amazon/opendistroforelasticsearch/sql/legacy/**/*IT.class' + } +} + +testClusters.docTest { + testDistribution = 'oss' + plugin file(tasks.getByPath(':plugin:bundlePlugin').archiveFile) +} + + +task comparisonTest(type: RestIntegTestTask) { + dependsOn ':plugin:bundlePlugin' + runner { + systemProperty 'tests.security.manager', 'false' + systemProperty('project.root', project.projectDir.absolutePath) + + // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for + // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time. + doFirst { systemProperty 'cluster.debug', getDebug()} + + if (System.getProperty("test.debug") != null) { + jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' + } + + include 'com/amazon/opendistroforelasticsearch/sql/correctness/**/*IT.class' + exclude 'com/amazon/opendistroforelasticsearch/sql/doctest/**/*IT.class' + exclude 'com/amazon/opendistroforelasticsearch/sql/ppl/**/*IT.class' + exclude 'com/amazon/opendistroforelasticsearch/sql/legacy/**/*IT.class' + + // Enable logging output to console + testLogging.showStandardStreams true + + // Pass down system properties to IT class + systemProperty "esHost", System.getProperty("esHost") + systemProperty "dbUrl", System.getProperty("dbUrl") + systemProperty "otherDbUrls", System.getProperty("otherDbUrls") + systemProperty "queries", System.getProperty("queries") + } +} + +testClusters.comparisonTest { + testDistribution = 'oss' + plugin file(tasks.getByPath(':plugin:bundlePlugin').archiveFile) +} + diff --git a/integ-test/lombok.config b/integ-test/lombok.config new file mode 100644 index 0000000000..6aa51d71ec --- /dev/null +++ b/integ-test/lombok.config @@ -0,0 +1,2 @@ +# This file is generated by the 'io.freefair.lombok' Gradle plugin +config.stopBubbling = true diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/CorrectnessIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/CorrectnessIT.java new file mode 100644 index 0000000000..8ab9f28e84 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/CorrectnessIT.java @@ -0,0 +1,174 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.correctness; + +import static com.amazon.opendistroforelasticsearch.sql.util.TestUtils.getResourceFilePath; + +import com.amazon.opendistroforelasticsearch.sql.correctness.report.TestReport; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.ComparisonTest; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection.DBConnection; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection.ESConnection; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection.JDBCConnection; +import com.amazon.opendistroforelasticsearch.sql.correctness.testset.TestDataSet; +import com.amazon.opendistroforelasticsearch.sql.legacy.CustomExternalTestCluster; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import com.google.common.collect.Maps; +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Map; +import java.util.TimeZone; +import org.apache.http.HttpHost; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.TestCluster; +import org.json.JSONObject; +import org.junit.Test; + +/** + * Correctness integration test by performing comparison test with other databases. + */ +@ESIntegTestCase.SuiteScopeTestCase +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numDataNodes = 3, supportsDedicatedMasters = false, transportClientRatio = 1) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class CorrectnessIT extends ESIntegTestCase { + + private static final Logger LOG = LogManager.getLogger(); + + @Test + public void performComparisonTest() { + TestConfig config = new TestConfig(getCmdLineArgs()); + LOG.info("Starting comparison test {}", config); + + try (ComparisonTest test = new ComparisonTest(getThisDBConnection(config), + getOtherDBConnections(config))) { + LOG.info("Loading test data set..."); + test.connect(); + for (TestDataSet dataSet : config.getTestDataSets()) { + test.loadData(dataSet); + } + + LOG.info("Verifying test queries..."); + TestReport report = test.verify(config.getTestQuerySet()); + + LOG.info("Saving test report to disk..."); + store(report); + + LOG.info("Cleaning up test data..."); + for (TestDataSet dataSet : config.getTestDataSets()) { + test.cleanUp(dataSet); + } + } + LOG.info("Completed comparison test."); + } + + private Map getCmdLineArgs() { + return Maps.fromProperties(System.getProperties()); + } + + private DBConnection getThisDBConnection(TestConfig config) { + String dbUrl = config.getDbConnectionUrl(); + if (dbUrl.isEmpty()) { + return getESConnection(config); + } + return new JDBCConnection("DB Tested", dbUrl); + } + + /** + * Use Elasticsearch cluster given on CLI arg or internal embedded in SQLIntegTestCase + */ + private DBConnection getESConnection(TestConfig config) { + RestClient client; + String esHost = config.getESHostUrl(); + if (esHost.isEmpty()) { + client = getRestClient(); + esHost = client.getNodes().get(0).getHost().toString(); + } else { + client = RestClient.builder(HttpHost.create(esHost)).build(); + } + return new ESConnection("jdbc:elasticsearch://" + esHost, client); + } + + /** + * Create database connection with database name and connect URL + */ + private DBConnection[] getOtherDBConnections(TestConfig config) { + return config.getOtherDbConnectionNameAndUrls(). + entrySet().stream(). + map(e -> new JDBCConnection(e.getKey(), e.getValue())). + toArray(DBConnection[]::new); + } + + private void store(TestReport report) { + try { + // Create reports folder if not exists + String folderPath = "reports/"; + Path path = Paths.get(getResourceFilePath(folderPath)); + if (Files.notExists(path)) { + Files.createDirectory(path); + } + + // Write to report file + String relFilePath = folderPath + reportFileName(); + String absFilePath = getResourceFilePath(relFilePath); + byte[] content = new JSONObject(report).toString(2).getBytes(); + + LOG.info("Report file location is {}", absFilePath); + Files.write(Paths.get(absFilePath), content); + } catch (Exception e) { + throw new IllegalStateException("Failed to store report file", e); + } + } + + private String reportFileName() { + SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd-HH"); + df.setTimeZone(TimeZone.getTimeZone("GMT")); + String dateTime = df.format(new Date()); + return "report_" + dateTime + ".json"; + } + + @Override + protected TestCluster buildTestCluster(Scope scope, long seed) throws IOException { + + String clusterAddresses = System.getProperty(TESTS_CLUSTER); + + if (Strings.hasLength(clusterAddresses)) { + String[] stringAddresses = clusterAddresses.split(","); + TransportAddress[] transportAddresses = new TransportAddress[stringAddresses.length]; + int i = 0; + for (String stringAddress : stringAddresses) { + URL url = new URL("http://" + stringAddress); + InetAddress inetAddress = InetAddress.getByName(url.getHost()); + transportAddresses[i++] = + new TransportAddress(new InetSocketAddress(inetAddress, url.getPort())); + } + return new CustomExternalTestCluster(createTempDir(), externalClusterClientSettings(), + transportClientPlugins(), transportAddresses); + } + return super.buildTestCluster(scope, seed); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/TestConfig.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/TestConfig.java new file mode 100644 index 0000000000..1d4f7308f9 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/TestConfig.java @@ -0,0 +1,156 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.correctness; + +import static java.util.stream.Collectors.joining; + +import com.amazon.opendistroforelasticsearch.sql.correctness.testset.TestDataSet; +import com.amazon.opendistroforelasticsearch.sql.correctness.testset.TestQuerySet; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; +import com.google.common.base.Charsets; +import com.google.common.io.Resources; +import java.net.URL; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +/** + * Test configuration parse the following information from command line arguments: + * 1) Test schema and data + * 2) Test queries + * 3) Elasticsearch connection URL + * 4) Other database connection URLs + */ +public class TestConfig { + + private static final String DEFAULT_TEST_QUERIES = "tableau_integration_tests.txt"; + private static final String DEFAULT_OTHER_DB_URLS = + "H2=jdbc:h2:mem:test;DB_CLOSE_DELAY=-1," + "SQLite=jdbc:sqlite::memory:"; + + private final TestDataSet[] testDataSets; + + private final TestQuerySet testQuerySet; + + private final String esHostUrl; + + /** + * Test against some database rather than Elasticsearch via our JDBC driver + */ + private final String dbConnectionUrl; + + private final Map otherDbConnectionNameAndUrls = new HashMap<>(); + + public TestConfig(Map cliArgs) { + testDataSets = buildDefaultTestDataSet(); // TODO: parse test data set argument + testQuerySet = buildTestQuerySet(cliArgs); + esHostUrl = cliArgs.getOrDefault("esHost", ""); + dbConnectionUrl = cliArgs.getOrDefault("dbUrl", ""); + + parseOtherDbConnectionInfo(cliArgs); + } + + public TestDataSet[] getTestDataSets() { + return testDataSets; + } + + public TestQuerySet getTestQuerySet() { + return testQuerySet; + } + + public String getESHostUrl() { + return esHostUrl; + } + + public String getDbConnectionUrl() { + return dbConnectionUrl; + } + + public Map getOtherDbConnectionNameAndUrls() { + return otherDbConnectionNameAndUrls; + } + + private TestDataSet[] buildDefaultTestDataSet() { + return new TestDataSet[] { + new TestDataSet("kibana_sample_data_flights", + readFile("kibana_sample_data_flights.json"), + readFile("kibana_sample_data_flights.csv")), + new TestDataSet("kibana_sample_data_ecommerce", + readFile("kibana_sample_data_ecommerce.json"), + readFile("kibana_sample_data_ecommerce.csv")), + }; + } + + private TestQuerySet buildTestQuerySet(Map cliArgs) { + String queryFilePath = cliArgs.getOrDefault("queries", ""); // Gradle set it empty always + if (queryFilePath.isEmpty()) { + queryFilePath = DEFAULT_TEST_QUERIES; + } + return new TestQuerySet(readFile(queryFilePath)); + } + + private void parseOtherDbConnectionInfo(Map cliArgs) { + String otherDbUrls = cliArgs.getOrDefault("otherDbUrls", ""); + if (otherDbUrls.isEmpty()) { + otherDbUrls = DEFAULT_OTHER_DB_URLS; + } + + for (String dbNameAndUrl : otherDbUrls.split(",")) { + int firstEq = dbNameAndUrl.indexOf('='); + String dbName = dbNameAndUrl.substring(0, firstEq); + String dbUrl = dbNameAndUrl.substring(firstEq + 1); + otherDbConnectionNameAndUrls.put(dbName, dbUrl); + } + } + + private static String readFile(String relativePath) { + try { + URL url = Resources.getResource("correctness/" + relativePath); + return Resources.toString(url, Charsets.UTF_8); + } catch (Exception e) { + throw new IllegalStateException("Failed to read test file [" + relativePath + "]"); + } + } + + @Override + public String toString() { + return "\n=================================\n" + + "Tested Database : " + esHostUrlToString() + '\n' + + "Other Databases :\n" + otherDbConnectionInfoToString() + '\n' + + "Test data set(s) :\n" + testDataSetsToString() + '\n' + + "Test query set : " + testQuerySet + '\n' + + "=================================\n"; + } + + private String testDataSetsToString() { + return Arrays.stream(testDataSets). + map(TestDataSet::toString). + collect(joining("\n")); + } + + private String esHostUrlToString() { + if (!dbConnectionUrl.isEmpty()) { + return dbConnectionUrl; + } + return esHostUrl.isEmpty() ? "(Use internal Elasticsearch in workspace)" : esHostUrl; + } + + private String otherDbConnectionInfoToString() { + return otherDbConnectionNameAndUrls.entrySet().stream(). + map(e -> StringUtils.format(" %s = %s", e.getKey(), e.getValue())). + collect(joining("\n")); + } + +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/ErrorTestCase.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/ErrorTestCase.java similarity index 84% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/ErrorTestCase.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/ErrorTestCase.java index 2aaf593548..61975f179d 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/ErrorTestCase.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/ErrorTestCase.java @@ -15,12 +15,12 @@ package com.amazon.opendistroforelasticsearch.sql.correctness.report; +import static com.amazon.opendistroforelasticsearch.sql.correctness.report.TestCaseReport.TestResult.FAILURE; + import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; -import static com.amazon.opendistroforelasticsearch.sql.correctness.report.TestCaseReport.TestResult.FAILURE; - /** * Report for test case that ends with an error. */ @@ -29,12 +29,14 @@ @Getter public class ErrorTestCase extends TestCaseReport { - /** Root cause of the error */ - private final String reason; + /** + * Root cause of the error + */ + private final String reason; - public ErrorTestCase(int id, String sql, String reason) { - super(id, sql, FAILURE); - this.reason = reason; - } + public ErrorTestCase(int id, String sql, String reason) { + super(id, sql, FAILURE); + this.reason = reason; + } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/FailedTestCase.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/FailedTestCase.java similarity index 61% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/FailedTestCase.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/FailedTestCase.java index 93a6dbaec2..d6a209bde7 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/FailedTestCase.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/FailedTestCase.java @@ -15,16 +15,16 @@ package com.amazon.opendistroforelasticsearch.sql.correctness.report; +import static com.amazon.opendistroforelasticsearch.sql.correctness.report.TestCaseReport.TestResult.FAILURE; + import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.DBResult; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; -import java.util.Comparator; -import java.util.List; - -import static com.amazon.opendistroforelasticsearch.sql.correctness.report.TestCaseReport.TestResult.FAILURE; - /** * Report for test case that fails due to inconsistent result set. */ @@ -33,13 +33,26 @@ @Getter public class FailedTestCase extends TestCaseReport { - /** Inconsistent result sets for reporting */ - private final List resultSets; - - public FailedTestCase(int id, String sql, List resultSets) { - super(id, sql, FAILURE); - this.resultSets = resultSets; - this.resultSets.sort(Comparator.comparing(DBResult::getDatabaseName)); - } + /** + * Inconsistent result sets for reporting + */ + private final List resultSets; + + /** + * Explain where the difference is caused the test failure. + */ + private final String explain; + + public FailedTestCase(int id, String sql, List resultSets) { + super(id, sql, FAILURE); + this.resultSets = resultSets; + this.resultSets.sort(Comparator.comparing(DBResult::getDatabaseName)); + + // Generate explanation by diff the first result with remaining + this.explain = resultSets.subList(1, resultSets.size()) + .stream() + .map(result -> resultSets.get(0).diff(result)) + .collect(Collectors.joining(", ")); + } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/SuccessTestCase.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/SuccessTestCase.java similarity index 92% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/SuccessTestCase.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/SuccessTestCase.java index 7653c22eb0..9de597b7e6 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/SuccessTestCase.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/SuccessTestCase.java @@ -15,12 +15,12 @@ package com.amazon.opendistroforelasticsearch.sql.correctness.report; +import static com.amazon.opendistroforelasticsearch.sql.correctness.report.TestCaseReport.TestResult.SUCCESS; + import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; -import static com.amazon.opendistroforelasticsearch.sql.correctness.report.TestCaseReport.TestResult.SUCCESS; - /** * Report for successful test case result. */ @@ -29,7 +29,7 @@ @Getter public class SuccessTestCase extends TestCaseReport { - public SuccessTestCase(int id, String sql) { - super(id, sql, SUCCESS); - } + public SuccessTestCase(int id, String sql) { + super(id, sql, SUCCESS); + } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/TestCaseReport.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/TestCaseReport.java similarity index 70% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/TestCaseReport.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/TestCaseReport.java index 42406adaa3..099c82cd9b 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/TestCaseReport.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/TestCaseReport.java @@ -15,12 +15,12 @@ package com.amazon.opendistroforelasticsearch.sql.correctness.report; +import static com.amazon.opendistroforelasticsearch.sql.correctness.report.TestCaseReport.TestResult.SUCCESS; + import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; -import static com.amazon.opendistroforelasticsearch.sql.correctness.report.TestCaseReport.TestResult.SUCCESS; - /** * Base class for different test result. */ @@ -28,26 +28,26 @@ @ToString public abstract class TestCaseReport { - public enum TestResult { - SUCCESS, FAILURE; - } + public enum TestResult { + SUCCESS, FAILURE; + } - @Getter - private final int id; + @Getter + private final int id; - @Getter - private final String sql; + @Getter + private final String sql; - private final TestResult result; + private final TestResult result; - public TestCaseReport(int id, String sql, TestResult result) { - this.id = id; - this.sql = sql; - this.result = result; - } + public TestCaseReport(int id, String sql, TestResult result) { + this.id = id; + this.sql = sql; + this.result = result; + } - public String getResult() { - return result == SUCCESS ? "Success" : "Failed"; - } + public String getResult() { + return result == SUCCESS ? "Success" : "Failed"; + } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/TestReport.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/TestReport.java similarity index 66% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/TestReport.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/TestReport.java index 8ef1c4264f..b2ccd108e3 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/TestReport.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/TestReport.java @@ -15,13 +15,12 @@ package com.amazon.opendistroforelasticsearch.sql.correctness.report; +import java.util.ArrayList; +import java.util.List; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; -import java.util.ArrayList; -import java.util.List; - /** * Test report class to generate JSON report. */ @@ -30,21 +29,22 @@ @Getter public class TestReport { - private final TestSummary summary = new TestSummary(); - - private final List tests = new ArrayList<>(); - - /** - * Add a test case report to the whole report. - * @param testCase report for a single test case - */ - public void addTestCase(TestCaseReport testCase) { - tests.add(testCase); - if ("Success".equals(testCase.getResult())) { - summary.addSuccess(); - } else { - summary.addFailure(); - } + private final TestSummary summary = new TestSummary(); + + private final List tests = new ArrayList<>(); + + /** + * Add a test case report to the whole report. + * + * @param testCase report for a single test case + */ + public void addTestCase(TestCaseReport testCase) { + tests.add(testCase); + if ("Success".equals(testCase.getResult())) { + summary.addSuccess(); + } else { + summary.addFailure(); } + } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/TestSummary.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/TestSummary.java similarity index 79% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/TestSummary.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/TestSummary.java index 4da3c832e7..6303f9eb22 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/TestSummary.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/TestSummary.java @@ -27,20 +27,20 @@ @Getter public class TestSummary { - private int total; + private int total; - private int success; + private int success; - private int failure; + private int failure; - public void addSuccess() { - success++; - total++; - } + public void addSuccess() { + success++; + total++; + } - public void addFailure() { - failure++; - total++; - } + public void addFailure() { + failure++; + total++; + } } diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/ComparisonTest.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/ComparisonTest.java new file mode 100644 index 0000000000..1470ed48bb --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/ComparisonTest.java @@ -0,0 +1,182 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.correctness.runner; + +import static com.google.common.collect.ObjectArrays.concat; + +import com.amazon.opendistroforelasticsearch.sql.correctness.report.ErrorTestCase; +import com.amazon.opendistroforelasticsearch.sql.correctness.report.FailedTestCase; +import com.amazon.opendistroforelasticsearch.sql.correctness.report.SuccessTestCase; +import com.amazon.opendistroforelasticsearch.sql.correctness.report.TestCaseReport; +import com.amazon.opendistroforelasticsearch.sql.correctness.report.TestReport; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection.DBConnection; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.DBResult; +import com.amazon.opendistroforelasticsearch.sql.correctness.testset.TestDataSet; +import com.amazon.opendistroforelasticsearch.sql.correctness.testset.TestQuerySet; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; + +/** + * Comparison test runner for query result correctness. + */ +public class ComparisonTest implements AutoCloseable { + + /** + * Next id for test case + */ + private int testCaseId = 1; + + /** + * Connection for database being tested + */ + private final DBConnection thisConnection; + + /** + * Database connections for reference databases + */ + private final DBConnection[] otherDbConnections; + + public ComparisonTest(DBConnection thisConnection, DBConnection[] otherDbConnections) { + this.thisConnection = thisConnection; + this.otherDbConnections = otherDbConnections; + + // Guarantee ordering of other database in comparison test + Arrays.sort(this.otherDbConnections, Comparator.comparing(DBConnection::getDatabaseName)); + } + + /** + * Open database connection. + */ + public void connect() { + for (DBConnection conn : concat(thisConnection, otherDbConnections)) { + conn.connect(); + } + } + + /** + * Create table and load test data. + * + * @param dataSet test data set + */ + public void loadData(TestDataSet dataSet) { + for (DBConnection conn : concat(thisConnection, otherDbConnections)) { + conn.create(dataSet.getTableName(), dataSet.getSchema()); + insertTestDataInBatch(conn, dataSet.getTableName(), dataSet.getDataRows()); + } + } + + /** + * Verify queries one by one by comparing between databases. + * + * @param querySet SQL queries + * @return Test result report + */ + public TestReport verify(TestQuerySet querySet) { + TestReport report = new TestReport(); + for (String sql : querySet) { + try { + DBResult esResult = thisConnection.select(sql); + report.addTestCase(compareWithOtherDb(sql, esResult)); + } catch (Exception e) { + report.addTestCase(new ErrorTestCase(nextId(), sql, + StringUtils.format("%s: %s", e.getClass().getSimpleName(), extractRootCause(e)))); + } + } + return report; + } + + /** + * Clean up test table. + * + * @param dataSet test data set + */ + public void cleanUp(TestDataSet dataSet) { + for (DBConnection conn : concat(thisConnection, otherDbConnections)) { + conn.drop(dataSet.getTableName()); + } + } + + @Override + public void close() { + for (DBConnection conn : concat(thisConnection, otherDbConnections)) { + try { + conn.close(); + } catch (Exception e) { + // Ignore + } + } + } + + /** + * Execute the query and compare with current result + */ + private TestCaseReport compareWithOtherDb(String sql, DBResult esResult) { + List mismatchResults = Lists.newArrayList(esResult); + StringBuilder reasons = new StringBuilder(); + for (int i = 0; i < otherDbConnections.length; i++) { + try { + DBResult otherDbResult = otherDbConnections[i].select(sql); + if (esResult.equals(otherDbResult)) { + return new SuccessTestCase(nextId(), sql); + } + + mismatchResults.add(otherDbResult); + + // Cannot find any database result match + if (i == otherDbConnections.length - 1) { + return new FailedTestCase(nextId(), sql, mismatchResults); + } + } catch (Exception e) { + // Ignore and move on to next database + reasons.append(extractRootCause(e)).append(";"); + } + } + + // Cannot find any database support this query + return new ErrorTestCase(nextId(), sql, "No other databases support this query: " + reasons); + } + + private int nextId() { + return testCaseId++; + } + + private void insertTestDataInBatch(DBConnection conn, String tableName, List testData) { + Iterator iterator = testData.iterator(); + String[] fieldNames = iterator.next(); // first row is header of column names + Iterators.partition(iterator, 100). + forEachRemaining(batch -> conn.insert(tableName, fieldNames, batch)); + } + + private String extractRootCause(Throwable e) { + while (e.getCause() != null) { + e = e.getCause(); + } + + if (e.getLocalizedMessage() != null) { + return e.getLocalizedMessage(); + } + if (e.getMessage() != null) { + return e.getMessage(); + } + return e.toString(); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/DBConnection.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/DBConnection.java new file mode 100644 index 0000000000..6779398be4 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/DBConnection.java @@ -0,0 +1,73 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection; + +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.DBResult; +import java.util.List; + +/** + * Abstraction for different databases. + */ +public interface DBConnection { + + /** + * @return database name + */ + String getDatabaseName(); + + /** + * Connect to database by opening a connection. + */ + void connect(); + + /** + * Create table with the schema. + * + * @param tableName table name + * @param schema schema json in ES mapping format + */ + void create(String tableName, String schema); + + /** + * Insert batch of data to database. + * + * @param tableName table name + * @param columnNames column names + * @param batch batch of rows + */ + void insert(String tableName, String[] columnNames, List batch); + + /** + * Fetch data from database. + * + * @param query SQL query + * @return result set + */ + DBResult select(String query); + + /** + * Drop table. + * + * @param tableName table name + */ + void drop(String tableName); + + /** + * Close the database connection. + */ + void close(); + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/ESConnection.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/ESConnection.java new file mode 100644 index 0000000000..4ce4a7bde2 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/ESConnection.java @@ -0,0 +1,113 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection; + +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.DBResult; +import java.io.IOException; +import java.util.List; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.json.JSONObject; + +/** + * Elasticsearch database connection for insertion. This class wraps JDBCConnection to delegate query method. + */ +public class ESConnection implements DBConnection { + + /** + * Connection via our Elasticsearch JDBC driver + */ + private final DBConnection connection; + + /** + * Native Elasticsearch REST client for operation unsupported by driver such as CREATE/INSERT + */ + private final RestClient client; + + public ESConnection(String connectionUrl, RestClient client) { + this.connection = new JDBCConnection("Elasticsearch", connectionUrl); + this.client = client; + } + + @Override + public String getDatabaseName() { + return "Elasticsearch"; + } + + @Override + public void connect() { + connection.connect(); + } + + @Override + public void create(String tableName, String schema) { + Request request = new Request("PUT", "/" + tableName); + request.setJsonEntity(schema); + performRequest(request); + } + + @Override + public void drop(String tableName) { + performRequest(new Request("DELETE", "/" + tableName)); + } + + @Override + public void insert(String tableName, String[] columnNames, List batch) { + Request request = new Request("POST", "/" + tableName + "/_bulk?refresh=true"); + request.setJsonEntity(buildBulkBody(columnNames, batch)); + performRequest(request); + } + + @Override + public DBResult select(String query) { + return connection.select(query); + } + + @Override + public void close() { + // Only close database connection and leave ES REST connection alone + // because it's initialized and manged by ES test base class. + connection.close(); + } + + private void performRequest(Request request) { + try { + Response response = client.performRequest(request); + int status = response.getStatusLine().getStatusCode(); + if (status != 200) { + throw new IllegalStateException("Failed to perform request. Error code: " + status); + } + } catch (IOException e) { + throw new IllegalStateException("Failed to perform request", e); + } + } + + private String buildBulkBody(String[] columnNames, List batch) { + StringBuilder body = new StringBuilder(); + for (String[] fieldValues : batch) { + JSONObject json = new JSONObject(); + for (int i = 0; i < columnNames.length; i++) { + json.put(columnNames[i], fieldValues[i]); + } + + body.append("{\"index\":{}}\n"). + append(json).append("\n"); + } + return body.toString(); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/JDBCConnection.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/JDBCConnection.java new file mode 100644 index 0000000000..ed731330b1 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/JDBCConnection.java @@ -0,0 +1,192 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection; + +import static java.util.stream.Collectors.joining; + +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.DBResult; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.Row; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; +import com.google.common.base.Strings; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.List; +import org.json.JSONObject; + +/** + * Database connection by JDBC driver. + */ +public class JDBCConnection implements DBConnection { + + private static final String SINGLE_QUOTE = "'"; + private static final String DOUBLE_QUOTE = "''"; + + /** + * Database name for display + */ + private final String databaseName; + + /** + * Database connection URL + */ + private final String connectionUrl; + + /** + * Current live connection + */ + private Connection connection; + + public JDBCConnection(String databaseName, String connectionUrl) { + this.databaseName = databaseName; + this.connectionUrl = connectionUrl; + } + + @Override + public void connect() { + try { + connection = DriverManager.getConnection(connectionUrl); + } catch (Exception e) { + throw new IllegalStateException("Failed to open connection", e); + } + } + + @Override + public String getDatabaseName() { + return databaseName; + } + + @Override + public void create(String tableName, String schema) { + try (Statement stmt = connection.createStatement()) { + String types = parseColumnNameAndTypesInSchemaJson(schema); + stmt.executeUpdate(StringUtils.format("CREATE TABLE %s(%s)", tableName, types)); + } catch (SQLException e) { + throw new IllegalStateException("Failed to create table [" + tableName + "]", e); + } + } + + @Override + public void drop(String tableName) { + try (Statement stmt = connection.createStatement()) { + stmt.executeUpdate("DROP TABLE " + tableName); + } catch (SQLException e) { + throw new IllegalStateException("Failed to drop table [" + tableName + "]", e); + } + } + + @Override + public void insert(String tableName, String[] columnNames, List batch) { + try (Statement stmt = connection.createStatement()) { + String names = String.join(",", columnNames); + for (String[] fieldValues : batch) { + stmt.addBatch(StringUtils.format( + "INSERT INTO %s(%s) VALUES (%s)", tableName, names, getValueList(fieldValues))); + } + stmt.executeBatch(); + } catch (SQLException e) { + throw new IllegalStateException("Failed to execute update", e); + } + } + + @Override + public DBResult select(String query) { + try (Statement stmt = connection.createStatement()) { + ResultSet resultSet = stmt.executeQuery(query); + DBResult result = new DBResult(databaseName); + populateMetaData(resultSet, result); + populateData(resultSet, result); + return result; + } catch (SQLException e) { + throw new IllegalStateException("Failed to execute query [" + query + "]", e); + } + } + + @Override + public void close() { + try { + connection.close(); + } catch (SQLException e) { + // Ignore + } + } + + /** + * Parse out type in schema json and convert to field name and type pairs for CREATE TABLE statement. + */ + private String parseColumnNameAndTypesInSchemaJson(String schema) { + JSONObject json = (JSONObject) new JSONObject(schema).query("/mappings/properties"); + return json.keySet().stream(). + map(colName -> colName + " " + mapToJDBCType(json.getJSONObject(colName).getString("type"))) + . + collect(joining(",")); + } + + private String getValueList(String[] fieldValues) { + return Arrays.stream(fieldValues). + map(val -> val.replace(SINGLE_QUOTE, DOUBLE_QUOTE)). + map(val -> SINGLE_QUOTE + val + SINGLE_QUOTE). + collect(joining(",")); + } + + private void populateMetaData(ResultSet resultSet, DBResult result) throws SQLException { + ResultSetMetaData metaData = resultSet.getMetaData(); + for (int i = 1; i <= metaData.getColumnCount(); i++) { + + // Use label name (alias) if present. + String colName = metaData.getColumnLabel(i); + if (Strings.isNullOrEmpty(colName)) { + colName = metaData.getColumnName(i); + } + result.addColumn(colName, metaData.getColumnTypeName(i)); + } + } + + private void populateData(ResultSet resultSet, DBResult result) throws SQLException { + while (resultSet.next()) { + Row row = new Row(); + for (int i = 1; i <= result.columnSize(); i++) { + row.add(resultSet.getObject(i)); + } + result.addRow(row); + } + } + + private String mapToJDBCType(String esType) { + switch (esType.toUpperCase()) { + case "KEYWORD": + case "TEXT": + return "VARCHAR"; + case "DATE": + return "TIMESTAMP"; + case "HALF_FLOAT": + return "FLOAT"; + default: + return esType; + } + } + + /** + * Setter for unit test mock + */ + public void setConnection(Connection connection) { + this.connection = connection; + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/DBResult.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/DBResult.java new file mode 100644 index 0000000000..c8036e1a75 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/DBResult.java @@ -0,0 +1,172 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset; + +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; +import com.google.common.collect.ImmutableSet; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.json.JSONPropertyName; + +/** + * Query result for equality comparison. Based on different type of query, such as query with/without ORDER BY and + * query with SELECT columns or just *, order of column and row may matter or not. So the internal data structure of this + * class is passed in from outside either list or set, hash map or linked hash map etc. + */ +@EqualsAndHashCode(exclude = "databaseName") +@ToString +public class DBResult { + + /** + * Possible types for floating point number + */ + private static final Set FLOAT_TYPES = ImmutableSet.of("FLOAT", "DOUBLE", "REAL"); + + /** + * Database name for display + */ + private final String databaseName; + + /** + * Column name and types from result set meta data + */ + @Getter + private final Collection schema; + + /** + * Data rows from result set + */ + private final Collection dataRows; + + /** + * By default treat both columns and data rows in order. This makes sense for typical query + * with specific column names in SELECT but without ORDER BY. + */ + public DBResult(String databaseName) { + this(databaseName, new ArrayList<>(), new HashSet<>()); + } + + public DBResult(String databaseName, Collection schema, Collection rows) { + this.databaseName = databaseName; + this.schema = schema; + this.dataRows = rows; + } + + public int columnSize() { + return schema.size(); + } + + public void addColumn(String name, String type) { + type = StringUtils.toUpper(type); + + // Ignore float type by assigning all type names string to it. + if (FLOAT_TYPES.contains(type)) { + type = FLOAT_TYPES.toString(); + } + schema.add(new Type(StringUtils.toUpper(name), type)); + } + + public void addRow(Row row) { + dataRows.add(row); + } + + @JSONPropertyName("database") + public String getDatabaseName() { + return databaseName; + } + + /** + * Flatten for simplifying json generated + */ + public Collection> getDataRows() { + return dataRows.stream().map(Row::getValues).collect(Collectors.toSet()); + } + + /** + * Explain the difference between this and other DB result which is helpful for + * troubleshooting in final test report. + * @param other other DB result + * @return explain the difference + */ + public String diff(DBResult other) { + String result = diffSchema(other); + if (result.isEmpty()) { + return diffDataRows(other); + } + return result; + } + + private String diffSchema(DBResult other) { + List thisSchema = new ArrayList<>(schema); + List otherSchema = new ArrayList<>(other.schema); + return diff("Schema type", thisSchema, otherSchema); + } + + private String diffDataRows(DBResult other) { + List thisRows = sort(dataRows); + List otherRows = sort(other.dataRows); + return diff("Data row", thisRows, otherRows); + } + + /** + * Check if two lists are same otherwise explain if size or any element + * is different at some position. + */ + private String diff(String name, List thisList, List otherList) { + if (thisList.size() != otherList.size()) { + return StringUtils.format("%s size is different: this=[%d], other=[%d]", + name, thisList.size(), otherList.size()); + } + + int diff = findFirstDifference(thisList, otherList); + if (diff >= 0) { + return StringUtils.format("%s at [%d] is different: this=[%s], other=[%s]", + name, diff, thisList.get(diff), otherList.get(diff)); + } + return ""; + } + + /** + * Find first different element with assumption that the lists given have same size + * and there is no NULL element inside. + */ + private static int findFirstDifference(List list1, List list2) { + for (int i = 0; i < list1.size(); i++) { + if (!list1.get(i).equals(list2.get(i))) { + return i; + } + } + return -1; + } + + /** + * Convert a collection to list and sort and return this new list. + */ + private static > List sort(Collection collection) { + ArrayList list = new ArrayList<>(collection); + Collections.sort(list); + return list; + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/Row.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/Row.java new file mode 100644 index 0000000000..d409bd81f2 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/Row.java @@ -0,0 +1,88 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; + +/** + * Row in result set. + */ +@EqualsAndHashCode +@ToString +@Getter +public class Row implements Comparable { + + private final Collection values; + + public Row() { + this(new ArrayList<>()); // values in order by default + } + + public Row(Collection values) { + this.values = values; + } + + public void add(Object value) { + values.add(roundFloatNum(value)); + } + + private Object roundFloatNum(Object value) { + if (value instanceof Float) { + BigDecimal decimal = BigDecimal.valueOf((Float) value).setScale(2, RoundingMode.CEILING); + value = decimal.doubleValue(); // Convert to double too + } else if (value instanceof Double) { + BigDecimal decimal = BigDecimal.valueOf((Double) value).setScale(2, RoundingMode.CEILING); + value = decimal.doubleValue(); + } + return value; + } + + @SuppressWarnings("unchecked") + @Override + public int compareTo(Row other) { + List thisObjects = new ArrayList<>(values); + List otherObjects = new ArrayList<>(other.values); + + for (int i = 0; i < thisObjects.size(); i++) { + Object thisObject = thisObjects.get(i); + Object otherObject = otherObjects.get(i); + + /* + * Only one is null, otherwise (both null or non-null) go ahead. + * Always consider NULL is greater which means NULL comes last in ASC and first in DESC + */ + if (thisObject == null ^ otherObject == null) { + return thisObject == null ? 1 : -1; + } + + if (thisObject instanceof Comparable) { + int result = ((Comparable) thisObject).compareTo(otherObject); + if (result != 0) { + return result; + } + } // Ignore incomparable field silently? + } + return 0; + } + +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/Type.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/Type.java similarity index 87% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/Type.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/Type.java index de5fd347dd..7dbafc39a8 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/Type.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/Type.java @@ -23,10 +23,14 @@ @Data public class Type { - /** Column name */ - private final String name; + /** + * Column name + */ + private final String name; - /** Column type */ - private final String type; + /** + * Column type + */ + private final String type; } diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/ComparisonTestTest.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/ComparisonTestTest.java new file mode 100644 index 0000000000..3a6e48278e --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/ComparisonTestTest.java @@ -0,0 +1,199 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.correctness.tests; + +import static java.util.Arrays.asList; +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.correctness.report.ErrorTestCase; +import com.amazon.opendistroforelasticsearch.sql.correctness.report.FailedTestCase; +import com.amazon.opendistroforelasticsearch.sql.correctness.report.SuccessTestCase; +import com.amazon.opendistroforelasticsearch.sql.correctness.report.TestReport; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.ComparisonTest; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection.DBConnection; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.DBResult; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.Row; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.Type; +import com.amazon.opendistroforelasticsearch.sql.correctness.testset.TestQuerySet; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +/** + * Tests for {@link ComparisonTest} + */ +@RunWith(MockitoJUnitRunner.class) +public class ComparisonTestTest { + + @Mock + private DBConnection esConnection; + + @Mock + private DBConnection otherDbConnection; + + private ComparisonTest correctnessTest; + + @Before + public void setUp() { + when(esConnection.getDatabaseName()).thenReturn("ES"); + when(otherDbConnection.getDatabaseName()).thenReturn("Other"); + correctnessTest = new ComparisonTest( + esConnection, new DBConnection[] {otherDbConnection} + ); + } + + @Test + public void testSuccess() { + when(esConnection.select(anyString())).thenReturn( + new DBResult("ES", asList(new Type("firstname", "text")), asList(new Row(asList("John")))) + ); + when(otherDbConnection.select(anyString())).thenReturn( + new DBResult("Other DB", asList(new Type("firstname", "text")), + asList(new Row(asList("John")))) + ); + + TestReport expected = new TestReport(); + expected.addTestCase(new SuccessTestCase(1, "SELECT * FROM accounts")); + TestReport actual = correctnessTest.verify(querySet("SELECT * FROM accounts")); + assertEquals(expected, actual); + } + + @Test + public void testFailureDueToInconsistency() { + DBResult esResult = + new DBResult("ES", asList(new Type("firstname", "text")), asList(new Row(asList("John")))); + DBResult otherDbResult = new DBResult("Other DB", asList(new Type("firstname", "text")), + asList(new Row(asList("JOHN")))); + when(esConnection.select(anyString())).thenReturn(esResult); + when(otherDbConnection.select(anyString())).thenReturn(otherDbResult); + + TestReport expected = new TestReport(); + expected.addTestCase( + new FailedTestCase(1, "SELECT * FROM accounts", asList(esResult, otherDbResult))); + TestReport actual = correctnessTest.verify(querySet("SELECT * FROM accounts")); + assertEquals(expected, actual); + } + + @Test + public void testSuccessFinally() { + DBConnection anotherDbConnection = mock(DBConnection.class); + when(anotherDbConnection.getDatabaseName()).thenReturn("Another"); + correctnessTest = new ComparisonTest( + esConnection, new DBConnection[] {otherDbConnection, anotherDbConnection} + ); + + DBResult esResult = + new DBResult("ES", asList(new Type("firstname", "text")), asList(new Row(asList("John")))); + DBResult otherDbResult = new DBResult("Other DB", asList(new Type("firstname", "text")), + asList(new Row(asList("JOHN")))); + DBResult anotherDbResult = new DBResult("Another DB", asList(new Type("firstname", "text")), + asList(new Row(asList("John")))); + when(esConnection.select(anyString())).thenReturn(esResult); + when(otherDbConnection.select(anyString())).thenReturn(otherDbResult); + when(anotherDbConnection.select(anyString())).thenReturn(anotherDbResult); + + TestReport expected = new TestReport(); + expected.addTestCase(new SuccessTestCase(1, "SELECT * FROM accounts")); + TestReport actual = correctnessTest.verify(querySet("SELECT * FROM accounts")); + assertEquals(expected, actual); + } + + @Test + public void testFailureDueToEventualInconsistency() { + DBConnection anotherDbConnection = mock(DBConnection.class); + when(anotherDbConnection.getDatabaseName()) + .thenReturn("ZZZ DB"); // Make sure this will be called after Other DB + correctnessTest = new ComparisonTest( + esConnection, new DBConnection[] {otherDbConnection, anotherDbConnection} + ); + + DBResult esResult = + new DBResult("ES", asList(new Type("firstname", "text")), asList(new Row(asList("John")))); + DBResult otherDbResult = new DBResult("Other DB", asList(new Type("firstname", "text")), + asList(new Row(asList("JOHN")))); + DBResult anotherDbResult = new DBResult("ZZZ DB", asList(new Type("firstname", "text")), + asList(new Row(asList("Hank")))); + when(esConnection.select(anyString())).thenReturn(esResult); + when(otherDbConnection.select(anyString())).thenReturn(otherDbResult); + when(anotherDbConnection.select(anyString())).thenReturn(anotherDbResult); + + TestReport expected = new TestReport(); + expected.addTestCase(new FailedTestCase(1, "SELECT * FROM accounts", + asList(esResult, otherDbResult, anotherDbResult))); + TestReport actual = correctnessTest.verify(querySet("SELECT * FROM accounts")); + assertEquals(expected, actual); + } + + @Test + public void testErrorDueToESException() { + when(esConnection.select(anyString())).thenThrow(new RuntimeException("All shards failure")); + + TestReport expected = new TestReport(); + expected.addTestCase( + new ErrorTestCase(1, "SELECT * FROM accounts", "RuntimeException: All shards failure")); + TestReport actual = correctnessTest.verify(querySet("SELECT * FROM accounts")); + assertEquals(expected, actual); + } + + @Test + public void testErrorDueToNoOtherDBSupportThisQuery() { + when(esConnection.select(anyString())).thenReturn( + new DBResult("ES", asList(new Type("firstname", "text")), asList(new Row(asList("John")))) + ); + when(otherDbConnection.select(anyString())) + .thenThrow(new RuntimeException("Unsupported feature")); + + TestReport expected = new TestReport(); + expected.addTestCase(new ErrorTestCase(1, "SELECT * FROM accounts", + "No other databases support this query: Unsupported feature;")); + TestReport actual = correctnessTest.verify(querySet("SELECT * FROM accounts")); + assertEquals(expected, actual); + } + + @Test + public void testSuccessWhenOneDBSupportThisQuery() { + DBConnection anotherDbConnection = mock(DBConnection.class); + when(anotherDbConnection.getDatabaseName()).thenReturn("Another"); + correctnessTest = new ComparisonTest( + esConnection, new DBConnection[] {otherDbConnection, anotherDbConnection} + ); + + when(esConnection.select(anyString())).thenReturn( + new DBResult("ES", asList(new Type("firstname", "text")), asList(new Row(asList("John")))) + ); + when(otherDbConnection.select(anyString())) + .thenThrow(new RuntimeException("Unsupported feature")); + when(anotherDbConnection.select(anyString())).thenReturn( + new DBResult("Another DB", asList(new Type("firstname", "text")), + asList(new Row(asList("John")))) + ); + + TestReport expected = new TestReport(); + expected.addTestCase(new SuccessTestCase(1, "SELECT * FROM accounts")); + TestReport actual = correctnessTest.verify(querySet("SELECT * FROM accounts")); + assertEquals(expected, actual); + } + + private TestQuerySet querySet(String query) { + return new TestQuerySet(new String[] {query}); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/DBResultTest.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/DBResultTest.java new file mode 100644 index 0000000000..ce52bcbf27 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/DBResultTest.java @@ -0,0 +1,92 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.correctness.tests; + +import static java.util.Collections.emptyList; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.DBResult; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.Row; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.Type; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.util.Arrays; +import org.junit.Test; + +/** + * Unit tests for {@link DBResult} + */ +public class DBResultTest { + + @Test + public void dbResultFromDifferentDbNameShouldEqual() { + DBResult result1 = + new DBResult("DB 1", Arrays.asList(new Type("name", "VARCHAR")), emptyList()); + DBResult result2 = + new DBResult("DB 2", Arrays.asList(new Type("name", "VARCHAR")), emptyList()); + assertEquals(result1, result2); + } + + @Test + public void dbResultWithDifferentColumnShouldNotEqual() { + DBResult result1 = + new DBResult("DB 1", Arrays.asList(new Type("name", "VARCHAR")), emptyList()); + DBResult result2 = new DBResult("DB 2", Arrays.asList(new Type("age", "INT")), emptyList()); + assertNotEquals(result1, result2); + } + + @Test + public void dbResultWithDifferentColumnTypeShouldNotEqual() { + DBResult result1 = new DBResult("DB 1", Arrays.asList(new Type("age", "FLOAT")), emptyList()); + DBResult result2 = new DBResult("DB 2", Arrays.asList(new Type("age", "INT")), emptyList()); + assertNotEquals(result1, result2); + } + + @Test + public void shouldExplainColumnTypeDifference() { + DBResult result1 = new DBResult("DB 1", + Arrays.asList(new Type("name", "VARCHAR"), new Type("age", "FLOAT")), emptyList()); + DBResult result2 = new DBResult("DB 2", + Arrays.asList(new Type("name", "VARCHAR"), new Type("age", "INT")), emptyList()); + + assertEquals( + "Schema type at [1] is different: " + + "this=[Type(name=age, type=FLOAT)], other=[Type(name=age, type=INT)]", + result1.diff(result2) + ); + } + + @Test + public void shouldExplainDataRowsDifference() { + DBResult result1 = new DBResult("DB 1", Arrays.asList(new Type("name", "VARCHAR")), + Sets.newHashSet( + new Row(Arrays.asList("hello")), + new Row(Arrays.asList("world")), + new Row(Lists.newArrayList((Object) null)))); + DBResult result2 = new DBResult("DB 2",Arrays.asList(new Type("name", "VARCHAR")), + Sets.newHashSet( + new Row(Lists.newArrayList((Object) null)), + new Row(Arrays.asList("hello")), + new Row(Arrays.asList("world123")))); + + assertEquals( + "Data row at [1] is different: this=[Row(values=[world])], other=[Row(values=[world123])]", + result1.diff(result2) + ); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/ESConnectionTest.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/ESConnectionTest.java new file mode 100644 index 0000000000..16a24375ff --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/ESConnectionTest.java @@ -0,0 +1,110 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.correctness.tests; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection.ESConnection; +import com.google.common.io.CharStreams; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Arrays; +import org.apache.http.ProtocolVersion; +import org.apache.http.message.BasicStatusLine; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +/** + * Tests for {@link ESConnection} + */ +@RunWith(MockitoJUnitRunner.class) +public class ESConnectionTest { + + @Mock + private RestClient client; + + private ESConnection conn; + + @Before + public void setUp() throws IOException { + conn = new ESConnection("jdbc:elasticsearch://localhost:12345", client); + + Response response = mock(Response.class); + when(client.performRequest(any(Request.class))).thenReturn(response); + when(response.getStatusLine()) + .thenReturn(new BasicStatusLine(new ProtocolVersion("HTTP", 2, 0), 200, "")); + } + + @Test + public void testCreateTable() throws IOException { + conn.create("test", "mapping"); + + Request actual = captureActualArg(); + assertEquals("PUT", actual.getMethod()); + assertEquals("/test", actual.getEndpoint()); + assertEquals("mapping", getBody(actual)); + } + + @Test + public void testInsertData() throws IOException { + conn.insert("test", new String[] {"name"}, + Arrays.asList(new String[] {"John"}, new String[] {"Hank"})); + + Request actual = captureActualArg(); + assertEquals("POST", actual.getMethod()); + assertEquals("/test/_bulk?refresh=true", actual.getEndpoint()); + assertEquals( + "{\"index\":{}}\n" + + "{\"name\":\"John\"}\n" + + "{\"index\":{}}\n" + + "{\"name\":\"Hank\"}\n", + getBody(actual) + ); + } + + @Test + public void testDropTable() throws IOException { + conn.drop("test"); + + Request actual = captureActualArg(); + assertEquals("DELETE", actual.getMethod()); + assertEquals("/test", actual.getEndpoint()); + } + + private Request captureActualArg() throws IOException { + ArgumentCaptor argCap = ArgumentCaptor.forClass(Request.class); + verify(client).performRequest(argCap.capture()); + return argCap.getValue(); + } + + private String getBody(Request request) throws IOException { + InputStream inputStream = request.getEntity().getContent(); + return CharStreams.toString(new InputStreamReader(inputStream)); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/JDBCConnectionTest.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/JDBCConnectionTest.java new file mode 100644 index 0000000000..9a616ab82b --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/JDBCConnectionTest.java @@ -0,0 +1,224 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.correctness.tests; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection.JDBCConnection; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.DBResult; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.Type; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.stubbing.OngoingStubbing; + +/** + * Tests for {@link JDBCConnection} + */ +@RunWith(MockitoJUnitRunner.class) +public class JDBCConnectionTest { + + @Mock + private Connection connection; + + @Mock + private Statement statement; + + private JDBCConnection conn; + + @Before + public void setUp() throws SQLException { + conn = new JDBCConnection("Test DB", "jdbc:testdb://localhost:12345"); + conn.setConnection(connection); + + when(connection.createStatement()).thenReturn(statement); + } + + @Test + public void testCreateTable() throws SQLException { + conn.create("test", + "{\"mappings\":{\"properties\":{\"name\":{\"type\":\"keyword\"},\"age\":{\"type\":\"INT\"}}}}"); + + ArgumentCaptor argCap = ArgumentCaptor.forClass(String.class); + verify(statement).executeUpdate(argCap.capture()); + String actual = argCap.getValue(); + + assertEquals("CREATE TABLE test(name VARCHAR,age INT)", actual); + } + + @Test + public void testDropTable() throws SQLException { + conn.drop("test"); + + ArgumentCaptor argCap = ArgumentCaptor.forClass(String.class); + verify(statement).executeUpdate(argCap.capture()); + String actual = argCap.getValue(); + + assertEquals("DROP TABLE test", actual); + } + + @Test + public void testInsertData() throws SQLException { + conn.insert("test", new String[] {"name", "age"}, + Arrays.asList(new String[] {"John", "25"}, new String[] {"Hank", "30"})); + + ArgumentCaptor argCap = ArgumentCaptor.forClass(String.class); + verify(statement, times(2)).addBatch(argCap.capture()); + List actual = argCap.getAllValues(); + + assertEquals( + Arrays.asList( + "INSERT INTO test(name,age) VALUES ('John','25')", + "INSERT INTO test(name,age) VALUES ('Hank','30')" + ), actual + ); + } + + @Test + public void testSelectQuery() throws SQLException { + ResultSetMetaData metaData = mockMetaData(ImmutableMap.of("name", "VARCHAR", "age", "INT")); + ResultSet resultSet = mockResultSet(new Object[] {"John", 25}, new Object[] {"Hank", 30}); + when(statement.executeQuery(anyString())).thenReturn(resultSet); + when(resultSet.getMetaData()).thenReturn(metaData); + + DBResult result = conn.select("SELECT * FROM test"); + assertEquals("Test DB", result.getDatabaseName()); + assertEquals( + Arrays.asList( + new Type("NAME", "VARCHAR"), + new Type("AGE", "INT") + ), + result.getSchema() + ); + assertEquals( + Sets.newHashSet( + Arrays.asList("John", 25), + Arrays.asList("Hank", 30) + ), + result.getDataRows() + ); + } + + @Test + public void testSelectQueryWithAlias() throws SQLException { + ResultSetMetaData metaData = + mockMetaData(ImmutableMap.of("name", "VARCHAR", "age", "INT"), "n", "a"); + ResultSet resultSet = mockResultSet(new Object[] {"John", 25}, new Object[] {"Hank", 30}); + when(statement.executeQuery(anyString())).thenReturn(resultSet); + when(resultSet.getMetaData()).thenReturn(metaData); + + DBResult result = conn.select("SELECT * FROM test"); + assertEquals( + Arrays.asList( + new Type("N", "VARCHAR"), + new Type("A", "INT") + ), + result.getSchema() + ); + } + + @Test + public void testSelectQueryWithFloatInResultSet() throws SQLException { + ResultSetMetaData metaData = + mockMetaData(ImmutableMap.of("name", "VARCHAR", "balance", "FLOAT")); + ResultSet resultSet = mockResultSet( + new Object[] {"John", 25.123}, + new Object[] {"Hank", 30.456}, + new Object[] {"Allen", 15.1} + ); + when(statement.executeQuery(anyString())).thenReturn(resultSet); + when(resultSet.getMetaData()).thenReturn(metaData); + + DBResult result = conn.select("SELECT * FROM test"); + assertEquals( + Arrays.asList( + new Type("NAME", "VARCHAR"), + new Type("BALANCE", "[FLOAT, DOUBLE, REAL]") + ), + result.getSchema() + ); + assertEquals( + Sets.newHashSet( + Arrays.asList("John", 25.13), + Arrays.asList("Hank", 30.46), + Arrays.asList("Allen", 15.1) + ), + result.getDataRows() + ); + } + + private ResultSet mockResultSet(Object[]... rows) throws SQLException { + ResultSet resultSet = mock(ResultSet.class); + OngoingStubbing next = when(resultSet.next()); + for (int i = 0; i < rows.length; i++) { + next = next.thenReturn(true); + } + next.thenReturn(false); + + OngoingStubbing getObject = when(resultSet.getObject(anyInt())); + for (Object[] row : rows) { + for (Object val : row) { + getObject = getObject.thenReturn(val); + } + } + return resultSet; + } + + private ResultSetMetaData mockMetaData(Map nameAndTypes, String... aliases) + throws SQLException { + ResultSetMetaData metaData = mock(ResultSetMetaData.class); + + OngoingStubbing getColumnName = when(metaData.getColumnName(anyInt())); + for (String name : nameAndTypes.keySet()) { + getColumnName = getColumnName.thenReturn(name); + } + + OngoingStubbing getColumnTypeName = when(metaData.getColumnTypeName(anyInt())); + for (String value : nameAndTypes.values()) { + getColumnTypeName = getColumnTypeName.thenReturn(value); + } + + if (aliases.length > 0) { + OngoingStubbing getColumnLabel = when(metaData.getColumnLabel(anyInt())); + for (String alias : aliases) { + getColumnLabel = getColumnLabel.thenReturn(alias); + } + } + + when(metaData.getColumnCount()).thenReturn(nameAndTypes.size()); + return metaData; + } + +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/RowTest.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/RowTest.java similarity index 56% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/RowTest.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/RowTest.java index 5992c975f1..28dbb480f8 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/RowTest.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/RowTest.java @@ -15,35 +15,46 @@ package com.amazon.opendistroforelasticsearch.sql.correctness.tests; -import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.Row; -import org.junit.Test; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.Row; +import org.junit.Test; + /** * Unit test {@link Row} */ public class RowTest { - @Test - public void rowShouldEqualToOtherRowWithSimilarFloat() { - Row row1 = new Row(); - Row row2 = new Row(); - row1.add(1.000001); - row2.add(1.000002); - assertEquals(row1, row2); - assertEquals(row2, row1); - } - - @Test - public void rowShouldNotEqualToOtherRowWithDifferentString() { - Row row1 = new Row(); - Row row2 = new Row(); - row1.add("hello"); - row2.add("hello1"); - assertNotEquals(row1, row2); - assertNotEquals(row2, row1); - } + @Test + public void rowShouldEqualToOtherRowWithSimilarFloat() { + Row row1 = new Row(); + Row row2 = new Row(); + row1.add(1.000001); + row2.add(1.000002); + assertEquals(row1, row2); + assertEquals(row2, row1); + } + + @Test + public void rowShouldNotEqualToOtherRowWithDifferentString() { + Row row1 = new Row(); + Row row2 = new Row(); + row1.add("hello"); + row2.add("hello1"); + assertNotEquals(row1, row2); + assertNotEquals(row2, row1); + } + + @Test + public void shouldConsiderNullGreater() { + Row row1 = new Row(); + Row row2 = new Row(); + row1.add("hello"); + row1.add(null); + row2.add("hello"); + row2.add("world"); + assertEquals(1, row1.compareTo(row2)); + } } diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestConfigTest.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestConfigTest.java new file mode 100644 index 0000000000..3ad5c6b0ed --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestConfigTest.java @@ -0,0 +1,71 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.correctness.tests; + +import static java.util.Collections.emptyMap; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.emptyString; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +import com.amazon.opendistroforelasticsearch.sql.correctness.TestConfig; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.junit.Test; + +/** + * Tests for {@link TestConfig} + */ +public class TestConfigTest { + + @Test + public void testDefaultConfig() { + TestConfig config = new TestConfig(emptyMap()); + assertThat(config.getESHostUrl(), is(emptyString())); + assertThat( + config.getOtherDbConnectionNameAndUrls(), + allOf( + hasEntry("H2", "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1"), + hasEntry("SQLite", "jdbc:sqlite::memory:") + ) + ); + } + + @Test + public void testCustomESUrls() { + Map args = ImmutableMap.of("esHost", "localhost:9200"); + TestConfig config = new TestConfig(args); + assertThat(config.getESHostUrl(), is("localhost:9200")); + } + + @Test + public void testCustomDbUrls() { + Map args = ImmutableMap.of("otherDbUrls", + "H2=jdbc:h2:mem:test;DB_CLOSE_DELAY=-1," + + "Derby=jdbc:derby:memory:myDb;create=true"); + + TestConfig config = new TestConfig(args); + assertThat( + config.getOtherDbConnectionNameAndUrls(), + allOf( + hasEntry("H2", "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1"), + hasEntry("Derby", "jdbc:derby:memory:myDb;create=true") + ) + ); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestDataSetTest.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestDataSetTest.java new file mode 100644 index 0000000000..6ca49fa34c --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestDataSetTest.java @@ -0,0 +1,71 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.correctness.tests; + +import static org.hamcrest.Matchers.contains; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; + +import com.amazon.opendistroforelasticsearch.sql.correctness.testset.TestDataSet; +import org.junit.Test; + +/** + * Tests for {@link TestDataSet} + */ +public class TestDataSetTest { + + @Test + public void testDataSetWithSingleColumnData() { + TestDataSet dataSet = new TestDataSet("test", "mappings", "hello\nworld\n123"); + assertEquals("test", dataSet.getTableName()); + assertEquals("mappings", dataSet.getSchema()); + assertThat( + dataSet.getDataRows(), + contains( + new String[] {"hello"}, + new String[] {"world"}, + new String[] {"123"} + ) + ); + } + + @Test + public void testDataSetWithMultiColumnsData() { + TestDataSet dataSet = new TestDataSet("test", "mappings", "hello,world\n123"); + assertThat( + dataSet.getDataRows(), + contains( + new String[] {"hello", "world"}, + new String[] {"123"} + ) + ); + } + + @Test + public void testDataSetWithEscapedComma() { + TestDataSet dataSet = new TestDataSet("test", "mappings", + "hello,\"hello,world,123\"\n123\n\"[abc,def,ghi]\",456"); + assertThat( + dataSet.getDataRows(), + contains( + new String[] {"hello", "hello,world,123"}, + new String[] {"123"}, + new String[] {"[abc,def,ghi]", "456"} + ) + ); + } + +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestQuerySetTest.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestQuerySetTest.java similarity index 73% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestQuerySetTest.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestQuerySetTest.java index 61b33d2927..f47dad0d42 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestQuerySetTest.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestQuerySetTest.java @@ -15,27 +15,28 @@ package com.amazon.opendistroforelasticsearch.sql.correctness.tests; -import com.amazon.opendistroforelasticsearch.sql.correctness.testset.TestQuerySet; -import org.junit.Test; - import static org.hamcrest.Matchers.contains; import static org.junit.Assert.assertThat; +import com.amazon.opendistroforelasticsearch.sql.correctness.testset.TestQuerySet; +import org.junit.Test; + /** * Tests for {@link TestQuerySet} */ public class TestQuerySetTest { - @Test - public void testQuerySet() { - TestQuerySet querySet = new TestQuerySet("SELECT * FROM accounts\nSELECT * FROM accounts LIMIT 5"); - assertThat( - querySet, - contains( - "SELECT * FROM accounts", - "SELECT * FROM accounts LIMIT 5" - ) - ); - } + @Test + public void testQuerySet() { + TestQuerySet querySet = + new TestQuerySet("SELECT * FROM accounts\nSELECT * FROM accounts LIMIT 5"); + assertThat( + querySet, + contains( + "SELECT * FROM accounts", + "SELECT * FROM accounts LIMIT 5" + ) + ); + } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestReportTest.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestReportTest.java similarity index 71% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestReportTest.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestReportTest.java index 7d15899d23..fa77a98a2b 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestReportTest.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestReportTest.java @@ -15,6 +15,10 @@ package com.amazon.opendistroforelasticsearch.sql.correctness.tests; +import static java.util.Arrays.asList; +import static java.util.Collections.singleton; +import static org.junit.Assert.fail; + import com.amazon.opendistroforelasticsearch.sql.correctness.report.ErrorTestCase; import com.amazon.opendistroforelasticsearch.sql.correctness.report.FailedTestCase; import com.amazon.opendistroforelasticsearch.sql.correctness.report.SuccessTestCase; @@ -25,23 +29,19 @@ import org.json.JSONObject; import org.junit.Test; -import static java.util.Arrays.asList; -import static java.util.Collections.singleton; -import static org.junit.Assert.fail; - /** * Test for {@link TestReport} */ public class TestReportTest { - private TestReport report = new TestReport(); + private TestReport report = new TestReport(); - @Test - public void testSuccessReport() { - report.addTestCase(new SuccessTestCase(1, "SELECT * FROM accounts")); - JSONObject actual = new JSONObject(report); - JSONObject expected = new JSONObject( - "{" + + @Test + public void testSuccessReport() { + report.addTestCase(new SuccessTestCase(1, "SELECT * FROM accounts")); + JSONObject actual = new JSONObject(report); + JSONObject expected = new JSONObject( + "{" + " \"summary\": {" + " \"total\": 1," + " \"success\": 1," + @@ -55,22 +55,24 @@ public void testSuccessReport() { " }" + " ]" + "}" - ); + ); - if (!actual.similar(expected)) { - fail("Actual JSON is different from expected: " + actual.toString(2)); - } + if (!actual.similar(expected)) { + fail("Actual JSON is different from expected: " + actual.toString(2)); } + } - @Test - public void testFailedReport() { - report.addTestCase(new FailedTestCase(1, "SELECT * FROM accounts", asList( - new DBResult("Elasticsearch", singleton(new Type("firstName", "text")), singleton(new Row(asList("hello")))), - new DBResult("H2", singleton(new Type("firstName", "text")), singleton(new Row(asList("world")))) - ))); - JSONObject actual = new JSONObject(report); - JSONObject expected = new JSONObject( - "{" + + @Test + public void testFailedReport() { + report.addTestCase(new FailedTestCase(1, "SELECT * FROM accounts", asList( + new DBResult("Elasticsearch", singleton(new Type("firstName", "text")), + singleton(new Row(asList("hello")))), + new DBResult("H2", singleton(new Type("firstName", "text")), + singleton(new Row(asList("world")))) + ))); + JSONObject actual = new JSONObject(report); + JSONObject expected = new JSONObject( + "{" + " \"summary\": {" + " \"total\": 1," + " \"success\": 0," + @@ -81,6 +83,7 @@ public void testFailedReport() { " \"id\": 1," + " \"result\": 'Failed'," + " \"sql\": \"SELECT * FROM accounts\"," + + " \"explain\": \"Data row at [0] is different: this=[Row(values=[hello])], other=[Row(values=[world])]\"," + " \"resultSets\": [" + " {" + " \"database\": \"Elasticsearch\"," + @@ -106,19 +109,19 @@ public void testFailedReport() { " }" + " ]" + "}" - ); + ); - if (!actual.similar(expected)) { - fail("Actual JSON is different from expected: " + actual.toString(2)); - } + if (!actual.similar(expected)) { + fail("Actual JSON is different from expected: " + actual.toString(2)); } + } - @Test - public void testErrorReport() { - report.addTestCase(new ErrorTestCase(1, "SELECT * FROM", "Missing table name in query")); - JSONObject actual = new JSONObject(report); - JSONObject expected = new JSONObject( - "{" + + @Test + public void testErrorReport() { + report.addTestCase(new ErrorTestCase(1, "SELECT * FROM", "Missing table name in query")); + JSONObject actual = new JSONObject(report); + JSONObject expected = new JSONObject( + "{" + " \"summary\": {" + " \"total\": 1," + " \"success\": 0," + @@ -133,11 +136,11 @@ public void testErrorReport() { " }" + " ]" + "}" - ); + ); - if (!actual.similar(expected)) { - fail("Actual JSON is different from expected: " + actual.toString(2)); - } + if (!actual.similar(expected)) { + fail("Actual JSON is different from expected: " + actual.toString(2)); } + } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/UnitTests.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/UnitTests.java similarity index 100% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/UnitTests.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/UnitTests.java diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/testset/TestDataSet.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/testset/TestDataSet.java new file mode 100644 index 0000000000..ab5b77dc09 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/testset/TestDataSet.java @@ -0,0 +1,98 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.correctness.testset; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils.unquoteSingleField; +import static java.util.stream.Collectors.joining; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Test data set + */ +public class TestDataSet { + + private final String tableName; + private final String schema; + private final List dataRows; + + public TestDataSet(String tableName, String schemaFileContent, String dataFileContent) { + this.tableName = tableName; + this.schema = schemaFileContent; + this.dataRows = splitColumns(dataFileContent, ','); + } + + public String getTableName() { + return tableName; + } + + public String getSchema() { + return schema; + } + + public List getDataRows() { + return dataRows; + } + + /** + * Split columns in each line by separator and ignore escaped separator(s) in quoted string. + */ + private List splitColumns(String content, char separator) { + List result = new ArrayList<>(); + for (String line : content.split("\\r?\\n")) { + + List columns = new ArrayList<>(); + boolean isQuoted = false; + int start = 0; + for (int i = 0; i < line.length(); i++) { + + char c = line.charAt(i); + if (c == separator) { + if (isQuoted) { // Ignore comma(s) in quoted string + continue; + } + + String column = line.substring(start, i); + columns.add(unquoteSingleField(column, "\"")); + start = i + 1; + + } else if (c == '\"') { + isQuoted = !isQuoted; + } + } + + columns.add(unquoteSingleField(line.substring(start), "\"")); + result.add(columns.toArray(new String[0])); + } + return result; + } + + @Override + public String toString() { + int total = dataRows.size(); + return "Test data set :\n" + + " Table name: " + tableName + '\n' + + " Schema: " + schema + '\n' + + " Data rows (first 5 in " + total + "):" + + dataRows.stream(). + limit(5). + map(Arrays::toString). + collect(joining("\n ", "\n ", "\n")); + } + +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/testset/TestQuerySet.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/testset/TestQuerySet.java similarity index 50% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/testset/TestQuerySet.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/testset/TestQuerySet.java index dbad112b2f..a563c635cd 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/testset/TestQuerySet.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/testset/TestQuerySet.java @@ -15,51 +15,53 @@ package com.amazon.opendistroforelasticsearch.sql.correctness.testset; +import static java.util.stream.Collectors.joining; + import java.util.Arrays; import java.util.Iterator; import java.util.List; -import static java.util.stream.Collectors.joining; - /** * Test query set including SQL queries for comparison testing. */ public class TestQuerySet implements Iterable { - private List queries; + private List queries; - /** - * Construct by a test query file. - * @param queryFileContent file content with query per line - */ - public TestQuerySet(String queryFileContent) { - queries = lines(queryFileContent); - } + /** + * Construct by a test query file. + * + * @param queryFileContent file content with query per line + */ + public TestQuerySet(String queryFileContent) { + queries = lines(queryFileContent); + } - /** - * Construct by a test query array. - * @param queries query in array - */ - public TestQuerySet(String[] queries) { - this.queries = Arrays.asList(queries); - } + /** + * Construct by a test query array. + * + * @param queries query in array + */ + public TestQuerySet(String[] queries) { + this.queries = Arrays.asList(queries); + } - @Override - public Iterator iterator() { - return queries.iterator(); - } + @Override + public Iterator iterator() { + return queries.iterator(); + } - private List lines(String content) { - return Arrays.asList(content.split("\\r?\\n")); - } + private List lines(String content) { + return Arrays.asList(content.split("\\r?\\n")); + } - @Override - public String toString() { - int total = queries.size(); - return "SQL queries (first 5 in " + total + "):" - + queries.stream(). - limit(5). - collect(joining("\n ", "\n ", "\n")); - } + @Override + public String toString() { + int total = queries.size(); + return "SQL queries (first 5 in " + total + "):" + + queries.stream(). + limit(5). + collect(joining("\n ", "\n ", "\n")); + } } diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/admin/MonitoringIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/admin/MonitoringIT.java new file mode 100644 index 0000000000..f9ce4d62a6 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/admin/MonitoringIT.java @@ -0,0 +1,85 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.doctest.admin; + +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.CURL_REQUEST; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.metrics.MetricName.DEFAULT_CURSOR_REQUEST_COUNT_TOTAL; +import static com.amazon.opendistroforelasticsearch.sql.legacy.metrics.MetricName.DEFAULT_CURSOR_REQUEST_TOTAL; +import static com.amazon.opendistroforelasticsearch.sql.legacy.metrics.MetricName.FAILED_REQ_COUNT_CB; +import static com.amazon.opendistroforelasticsearch.sql.legacy.metrics.MetricName.FAILED_REQ_COUNT_CUS; +import static com.amazon.opendistroforelasticsearch.sql.legacy.metrics.MetricName.FAILED_REQ_COUNT_SYS; +import static com.amazon.opendistroforelasticsearch.sql.legacy.metrics.MetricName.REQ_COUNT_TOTAL; +import static com.amazon.opendistroforelasticsearch.sql.legacy.metrics.MetricName.REQ_TOTAL; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlStatsAction.STATS_API_ENDPOINT; + +import com.amazon.opendistroforelasticsearch.sql.doctest.core.DocTest; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.DocTestConfig; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.Requests; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequest; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.response.DataTable; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.MetricName; + +/** + * Doc test for plugin monitoring functionality + */ +@DocTestConfig(template = "admin/monitoring.rst") +public class MonitoringIT extends DocTest { + + @Section + public void nodeStats() { + section( + title("Node Stats"), + description( + "The meaning of fields in the response is as follows:\n\n" + fieldDescriptions()), + example( + description(), + getStats(), + queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), + explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) + ) + ); + } + + private String fieldDescriptions() { + DataTable table = new DataTable(new String[] {"Field name", "Description"}); + table.addRow(row(REQ_TOTAL, "Total count of request")); + table.addRow(row(REQ_COUNT_TOTAL, "Total count of request within the interval")); + table.addRow(row(DEFAULT_CURSOR_REQUEST_TOTAL, "Total count of simple cursor request")); + table.addRow(row(DEFAULT_CURSOR_REQUEST_COUNT_TOTAL, + "Total count of simple cursor request within the interval")); + table.addRow(row(FAILED_REQ_COUNT_SYS, + "Count of failed request due to system error within the interval")); + table.addRow(row(FAILED_REQ_COUNT_CUS, + "Count of failed request due to bad request within the interval")); + table.addRow( + row(FAILED_REQ_COUNT_CB, "Indicate if plugin is being circuit broken within the interval")); + + return table.toString(); + } + + private String[] row(MetricName name, String description) { + return new String[] {name.getName(), description}; + } + + private Requests getStats() { + return new Requests(restClient(), new SqlRequest("GET", STATS_API_ENDPOINT, "")); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/admin/PluginSettingIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/admin/PluginSettingIT.java new file mode 100644 index 0000000000..2ee7e92ee0 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/admin/PluginSettingIT.java @@ -0,0 +1,222 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.doctest.admin; + +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.CURL_REQUEST; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.CURSOR_ENABLED; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.CURSOR_FETCH_SIZE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.CURSOR_KEEPALIVE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_ENABLED; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_SUGGESTION; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_THRESHOLD; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.QUERY_RESPONSE_FORMAT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.QUERY_SLOWLOG; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.SQL_ENABLED; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.SQL_NEW_ENGINE_ENABLED; +import static org.elasticsearch.common.settings.Setting.Property; +import static org.elasticsearch.common.settings.Setting.Property.Dynamic; +import static org.elasticsearch.common.settings.Setting.Property.Final; +import static org.elasticsearch.common.settings.Setting.Property.IndexScope; +import static org.elasticsearch.common.settings.Setting.Property.NodeScope; +import static org.elasticsearch.common.settings.Settings.EMPTY; + +import com.amazon.opendistroforelasticsearch.sql.doctest.core.DocTest; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.DocTestConfig; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.Example; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.ListItems; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.Format; +import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.stream.Collectors; +import org.elasticsearch.common.settings.Setting; + +/** + * Doc test for plugin settings. + */ +@DocTestConfig(template = "admin/settings.rst", testData = {"accounts.json"}) +public class PluginSettingIT extends DocTest { + + private static final SqlSettings SETTINGS = new SqlSettings(); + + @Section(1) + public void sqlEnabledSetting() { + docSetting( + SQL_ENABLED, + "You can disable SQL plugin to reject all coming requests.", + false, + "SELECT * FROM accounts" + ); + } + + @Section(2) + public void slowLogSetting() { + docSetting( + QUERY_SLOWLOG, + "You can configure the time limit (seconds) for slow query which would be logged as " + + "'Slow query: elapsed=xxx (ms)' in elasticsearch.log.", + 10 + ); + } + + @Section(3) + public void queryAnalysisEnabledSetting() { + docSetting( + QUERY_ANALYSIS_ENABLED, + "You can disable query analyzer to bypass strict syntactic and semantic analysis.", + false + ); + } + + @Section(4) + public void semanticSuggestionSetting() { + docSetting( + QUERY_ANALYSIS_SEMANTIC_SUGGESTION, + "You can enable query analyzer to suggest correct field names for quick fix.", + true, + "SELECT first FROM accounts" + ); + } + + @Section(5) + public void semanticAnalysisThresholdSetting() { + docSetting( + QUERY_ANALYSIS_SEMANTIC_THRESHOLD, + "Because query analysis needs to build semantic context in memory, index with large number of field " + + "would be skipped. You can update it to apply analysis to smaller or larger index as needed.", + 50 + ); + } + + @Section(6) + public void responseFormatSetting() { + docSetting( + QUERY_RESPONSE_FORMAT, + String.format("User can set default response format of the query. " + + "The supported format includes: %s.", Arrays.stream(Format.values()) + .map(Format::getFormatName) + .collect(Collectors.joining(","))), + Format.JSON.getFormatName(), + "SELECT firstname, lastname, age FROM accounts ORDER BY age LIMIT 2" + ); + } + + @Section(7) + public void cursorEnabledSetting() { + docSetting( + CURSOR_ENABLED, + "User can enable/disable pagination for all queries that are supported.", + true + ); + } + + @Section(8) + public void cursorDefaultFetchSizeSetting() { + docSetting( + CURSOR_FETCH_SIZE, + "User can set the default fetch_size for all queries that are supported by pagination. " + + "Explicit `fetch_size` passed in request will override this value", + 50 + ); + } + + @Section(9) + public void cursorDefaultContextKeepAliveSetting() { + docSetting( + CURSOR_KEEPALIVE, + "User can set this value to indicate how long the cursor context should be kept open. " + + "Cursor contexts are resource heavy, and a lower value should be used if possible.", + "5m" + ); + } + + @Section(10) + public void sqlNewQueryEngineSetting() { + docSetting( + SQL_NEW_ENGINE_ENABLED, + "We are migrating existing functionalities to a new query engine under development. " + + "User can choose to enable the new engine if interested or disable if any issue found.", + true + ); + } + + /** + * Generate content for sample queries with setting changed to new value. + * Finally setting will be reverted to avoid potential impact on other test cases. + */ + private void docSetting(String name, String description, Object sampleValue, + String... sampleQueries) { + try { + section( + title(name), + description(description + "\n\n" + listSettingDetails(name)), + createExamplesForSettingChangeQueryAndOtherSampleQueries(name, sampleValue, sampleQueries) + ); + } finally { + // Make sure the change is removed + try { + put(name, null).queryResponse(); + } catch (Exception e) { + // Ignore + } + } + } + + private Example[] createExamplesForSettingChangeQueryAndOtherSampleQueries(String name, + Object sampleValue, + String[] sampleQueries) { + Example[] examples = new Example[sampleQueries.length + 1]; + examples[0] = example("You can update the setting with a new value like this.", + put(name, sampleValue), + queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), + explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE)); + + for (int i = 0; i < sampleQueries.length; i++) { + examples[i + 1] = example("Query result after the setting updated is like:", + post(sampleQueries[i]), + queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), + explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE)); + } + return examples; + } + + private String listSettingDetails(String name) { + Setting setting = SETTINGS.getSetting(name); + ListItems list = new ListItems(); + + list.addItem(StringUtils.format("The default value is %s.", setting.getDefault(EMPTY))); + + EnumSet properties = setting.getProperties(); + if (properties.contains(NodeScope)) { + list.addItem("This setting is node scope."); + } else if (properties.contains(IndexScope)) { + list.addItem("This setting is index scope."); + } + + if (properties.contains(Dynamic)) { + list.addItem("This setting can be updated dynamically."); + } else if (properties.contains(Final)) { + list.addItem("This setting is not updatable."); + } + return list.toString(); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/beyond/FullTextIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/beyond/FullTextIT.java new file mode 100644 index 0000000000..a23b00ef5d --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/beyond/FullTextIT.java @@ -0,0 +1,146 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.doctest.beyond; + +import com.amazon.opendistroforelasticsearch.sql.doctest.core.DocTest; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.DocTestConfig; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; + +@DocTestConfig(template = "beyond/fulltext.rst", testData = {"accounts.json"}) +public class FullTextIT extends DocTest { + + @Section(1) + public void matchQuery() { + section( + title("Match Query"), + description( + "Match query is the standard query for full-text search in Elasticsearch. Both ``MATCHQUERY`` and", + "``MATCH_QUERY`` are functions for performing match query." + ), + example( + description( + "Both functions can accept field name as first argument and a text as second argument."), + post(multiLine( + "SELECT account_number, address", + "FROM accounts", + "WHERE MATCH_QUERY(address, 'Holmes')" + )) + ), + example( + description( + "Both functions can also accept single argument and be used in the following manner."), + post(multiLine( + "SELECT account_number, address", + "FROM accounts", + "WHERE address = MATCH_QUERY('Holmes')" + )) + ) + ); + } + + @Section(2) + public void multiMatchQuery() { + section( + title("Multi-match Query"), + description( + "Besides match query against a single field, you can search for a text with multiple fields.", + "Function ``MULTI_MATCH``, ``MULTIMATCH`` and ``MULTIMATCHQUERY`` are provided for this." + ), + example( + description( + "Each preceding function accepts ``query`` for a text and ``fields`` for field names or pattern", + "that the text given is searched against. For example, the following query is searching for", + "documents in index accounts with 'Dale' as either firstname or lastname." + ), + post(multiLine( + "SELECT firstname, lastname", + "FROM accounts", + "WHERE MULTI_MATCH('query'='Dale', 'fields'='*name')" + )) + ) + ); + } + + @Section(3) + public void queryStringQuery() { + section( + title("Query String Query"), + description( + "Query string query parses and splits a query string provided based on Lucene query string syntax.", + "The mini language supports logical connectives, wildcard, regex and proximity search. Please refer", + "to official documentation for more details. Note that an error is thrown in the case of any invalid", + "syntax in query string." + ), + example( + description( + "``QUERY`` function accepts query string and returns true or false respectively for document", + "that matches the query string or not." + ), + post(multiLine( + "SELECT account_number, address", + "FROM accounts", + "WHERE QUERY('address:Lane OR address:Street')" + )) + ) + ); + } + + @Section(4) + public void matchPhraseQuery() { + section( + title("Match Phrase Query"), + description( + "Match phrase query is similar to match query but it is used for matching exact phrases.", + "``MATCHPHRASE``, ``MATCH_PHRASE`` and ``MATCHPHRASEQUERY`` are provided for this purpose." + ), + example( + description(), + post(multiLine( + "SELECT account_number, address", + "FROM accounts", + "WHERE MATCH_PHRASE(address, '880 Holmes Lane')" + )) + ) + ); + } + + @Section(5) + public void scoreQuery() { + section( + title("Score Query"), + description( + "Elasticsearch supports to wrap a filter query so as to return a relevance score along with", + "every matching document. ``SCORE``, ``SCOREQUERY`` and ``SCORE_QUERY`` can be used for this." + ), + example( + description( + "The first argument is a match query expression and the second argument is for an optional", + "floating point number to boost the score. The default value is 1.0. Apart from this, an", + "implicit variable ``_score`` is available so you can return score for each document or", + "use it for sorting." + ), + post(multiLine( + "SELECT account_number, address, _score", + "FROM accounts", + "WHERE SCORE(MATCH_QUERY(address, 'Lane'), 0.5) OR", + " SCORE(MATCH_QUERY(address, 'Street'), 100)", + "ORDER BY _score" + )) + ) + ); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/beyond/PartiQLIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/beyond/PartiQLIT.java new file mode 100644 index 0000000000..7fc7ad97ed --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/beyond/PartiQLIT.java @@ -0,0 +1,154 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.doctest.beyond; + +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.TestData.TEST_DATA_FOLDER_ROOT; +import static com.amazon.opendistroforelasticsearch.sql.util.TestUtils.getResourceFilePath; + +import com.amazon.opendistroforelasticsearch.sql.doctest.core.DocTest; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.DocTestConfig; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.Example; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.JsonPrettyFormatter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +@DocTestConfig(template = "beyond/partiql.rst", testData = {"employees_nested.json"}) +public class PartiQLIT extends DocTest { + + @Section(1) + public void showTestData() { + section( + title("Test Data"), + description( + "The test index ``employees_nested`` used by all examples in this document is very similar to", + "the one used in official PartiQL documentation." + ), + createDummyExampleForTestData("employees_nested.json") + ); + } + + @Section(2) + public void queryNestedCollection() { + section( + title("Querying Nested Collection"), + description( + "In SQL-92, a database table can only have tuples that consists of scalar values.", + "PartiQL extends SQL-92 to allow you query and unnest nested collection conveniently.", + "In Elasticsearch world, this is very useful for index with object or nested field." + ), + example( + title("Unnesting a Nested Collection"), + description( + "In the following example, it finds nested document (project) with field value (name)", + "that satisfies the predicate (contains 'security'). Note that because each parent document", + "can have more than one nested documents, the matched nested document is flattened. In other", + "word, the final result is the Cartesian Product between parent and nested documents." + ), + post(multiLine( + "SELECT e.name AS employeeName,", + " p.name AS projectName", + "FROM employees_nested AS e,", + " e.projects AS p", + "WHERE p.name LIKE '%security%'" + )) + ), + /* + Issue: https://github.com/opendistro-for-elasticsearch/sql/issues/397 + example( + title("Preserving Parent Information with LEFT JOIN"), + description( + "The query in the preceding example is very similar to traditional join queries, except ``ON`` clause missing.", + "This is because it is implicitly in the nesting of nested documents (projects) into parent (employee). Therefore,", + "you can use ``LEFT JOIN`` to preserve the information in parent document associated." + ), + post( + "SELECT e.id AS id, " + + " e.name AS employeeName, " + + " e.title AS title, " + + " p.name AS projectName " + + "FROM employees_nested AS e " + + "LEFT JOIN e.projects AS p" + ) + )*/ + example( + title("Unnesting in Existential Subquery"), + description( + "Alternatively, a nested collection can be unnested in subquery to check if it", + "satisfies a condition." + ), + post(multiLine( + "SELECT e.name AS employeeName", + "FROM employees_nested AS e", + "WHERE EXISTS (", + " SELECT *", + " FROM e.projects AS p", + " WHERE p.name LIKE '%security%'", + ")" + )) + )/*, + Issue: https://github.com/opendistro-for-elasticsearch/sql/issues/398 + example( + title("Aggregating over a Nested Collection"), + description( + "After unnested, a nested collection can be aggregated just like a regular field." + ), + post(multiLine( + "SELECT", + " e.name AS employeeName,", + " COUNT(p) AS cnt", + "FROM employees_nested AS e,", + " e.projects AS p", + "WHERE p.name LIKE '%security%'", + "GROUP BY e.id, e.name", + "HAVING COUNT(p) >= 1" + ) + )) + */ + ); + } + + private Example createDummyExampleForTestData(String fileName) { + Example example = new Example(); + example.setTitle("Employees"); + example.setDescription(""); + example.setResult(parseJsonFromTestData(fileName)); + return example; + } + + /** + * Concat and pretty format document at odd number line in bulk request file + */ + private String parseJsonFromTestData(String fileName) { + Path path = Paths.get(getResourceFilePath(TEST_DATA_FOLDER_ROOT + fileName)); + try { + List lines = Files.readAllLines(path); + String json = IntStream.range(0, lines.size()). + filter(i -> i % 2 == 1). + mapToObj(lines::get). + collect(Collectors.joining(",", "{\"employees\":[", "]}")); + return JsonPrettyFormatter.format(json); + } catch (IOException e) { + throw new IllegalStateException("Failed to load test data: " + path, e); + } + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/DocTest.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/DocTest.java new file mode 100644 index 0000000000..94fdc41209 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/DocTest.java @@ -0,0 +1,145 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.doctest.core; + +import static com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope; +import static java.nio.file.StandardOpenOption.APPEND; +import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; + +import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.DocTestConfig; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.DocBuilder; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.markup.Document; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.markup.RstDocument; +import com.amazon.opendistroforelasticsearch.sql.legacy.CustomExternalTestCluster; +import com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils; +import com.carrotsearch.randomizedtesting.AnnotatedMethodProvider; +import com.carrotsearch.randomizedtesting.TestMethodAndParams; +import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering; +import com.carrotsearch.randomizedtesting.annotations.TestMethodProviders; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import java.io.IOException; +import java.io.PrintWriter; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Comparator; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.TestCluster; + +/** + * Documentation test base class + */ +@TestMethodProviders({DocTest.SectionMethod.class}) +@TestCaseOrdering(DocTest.SectionOrder.class) +@ESIntegTestCase.SuiteScopeTestCase +@ClusterScope(scope = SUITE, numDataNodes = 1, supportsDedicatedMasters = false, transportClientRatio = 1) +@ThreadLeakScope(Scope.NONE) +public abstract class DocTest extends ESIntegTestCase implements DocBuilder { + + @Override + protected void setupSuiteScopeCluster() { + DocTestConfig config = getClass().getAnnotation(DocTestConfig.class); + loadTestData(config); + copyTemplateToDocument(config); + } + + @Override + public RestClient restClient() { + return getRestClient(); + } + + @Override + public Document openDocument() { + DocTestConfig config = getClass().getAnnotation(DocTestConfig.class); + Path docPath = absolutePath(config.template()); + try { + PrintWriter docWriter = new PrintWriter(Files.newBufferedWriter(docPath, APPEND)); + return new RstDocument(docWriter); + } catch (IOException e) { + throw new IllegalStateException("Failed to open document file " + docPath, e); + } + } + + private void loadTestData(DocTestConfig config) { + String[] testFilePaths = config.testData(); + TestData testData = new TestData(testFilePaths); + testData.loadToES(this); + } + + private void copyTemplateToDocument(DocTestConfig config) { + Path docPath = absolutePath(config.template()); + Template template = new Template(config.template()); + template.copyToDocument(docPath); + } + + /** + * Method annotated by {@link Section} will be treated as test method. + */ + public static class SectionMethod extends AnnotatedMethodProvider { + public SectionMethod() { + super(Section.class); + } + } + + /** + * Test methods will execute in order defined by value in {@link Section} annotation. + */ + public static class SectionOrder implements Comparator { + @Override + public int compare(TestMethodAndParams method1, TestMethodAndParams method2) { + return Integer.compare(order(method1), order(method2)); + } + + private int order(TestMethodAndParams method) { + Section section = method.getTestMethod().getAnnotation(Section.class); + return section.value(); + } + } + + private Path absolutePath(String templateRelativePath) { + return Paths.get(TestUtils.getResourceFilePath(DOCUMENT_FOLDER_ROOT + templateRelativePath)); + } + + @Override + protected TestCluster buildTestCluster(Scope scope, long seed) throws IOException { + + String clusterAddresses = System.getProperty(TESTS_CLUSTER); + + if (Strings.hasLength(clusterAddresses)) { + String[] stringAddresses = clusterAddresses.split(","); + TransportAddress[] transportAddresses = new TransportAddress[stringAddresses.length]; + int i = 0; + for (String stringAddress : stringAddresses) { + URL url = new URL("http://" + stringAddress); + InetAddress inetAddress = InetAddress.getByName(url.getHost()); + transportAddresses[i++] = + new TransportAddress(new InetSocketAddress(inetAddress, url.getPort())); + } + return new CustomExternalTestCluster(createTempDir(), externalClusterClientSettings(), + transportClientPlugins(), transportAddresses); + } + return super.buildTestCluster(scope, seed); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/Template.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/Template.java new file mode 100644 index 0000000000..3f1a14834b --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/Template.java @@ -0,0 +1,58 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.doctest.core; + +import static java.nio.file.StandardCopyOption.COPY_ATTRIBUTES; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.util.TestUtils; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Abstraction for document template file + */ +public class Template { + + private static final String TEMPLATE_FOLDER_ROOT = "src/test/resources/doctest/templates/"; + + private final Path templateFullPath; + + public Template(String templateRelativePath) { + this.templateFullPath = + Paths.get(TestUtils.getResourceFilePath(TEMPLATE_FOLDER_ROOT + templateRelativePath)); + } + + /** + * Copy template file to target document. Replace it if existing. + * + * @param docFullPath full path of target document + */ + public void copyToDocument(Path docFullPath) { + try { + Files.createDirectories(docFullPath.getParent()); + Files.copy(templateFullPath, docFullPath, REPLACE_EXISTING, COPY_ATTRIBUTES); + } catch (IOException e) { + throw new IllegalStateException(StringUtils.format( + "Failed to copy from template [%s] to document file [%s]", templateFullPath, docFullPath), + e); + } + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/TestData.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/TestData.java new file mode 100644 index 0000000000..0c5d0667f8 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/TestData.java @@ -0,0 +1,81 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.doctest.core; + +import static com.amazon.opendistroforelasticsearch.sql.util.TestUtils.createIndexByRestClient; +import static com.amazon.opendistroforelasticsearch.sql.util.TestUtils.getResourceFilePath; +import static com.amazon.opendistroforelasticsearch.sql.util.TestUtils.loadDataByRestClient; + +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Test data for document generation + */ +public class TestData { + + public static final String MAPPINGS_FOLDER_ROOT = "src/test/resources/doctest/mappings/"; + public static final String TEST_DATA_FOLDER_ROOT = "src/test/resources/doctest/testdata/"; + + private final String[] testFilePaths; + + public TestData(String[] testFilePaths) { + this.testFilePaths = testFilePaths; + } + + /** + * Load test data in file to Elaticsearch cluster via client. + * + * @param test current test instance + */ + public void loadToES(DocTest test) { + for (String filePath : testFilePaths) { + String indexName = indexName(filePath); + try { + createIndexByRestClient(test.restClient(), indexName, getIndexMapping(filePath)); + loadDataByRestClient(test.restClient(), indexName, TEST_DATA_FOLDER_ROOT + filePath); + } catch (Exception e) { + throw new IllegalStateException(StringUtils.format( + "Failed to load mapping and test filePath from %s", filePath), e); + } + test.ensureGreen(indexName); + } + } + + /** + * Use file name (without file extension) as index name implicitly. + * For example, for 'testdata/accounts.json', 'accounts' will be used. + */ + private String indexName(String filePath) { + return filePath.substring( + filePath.lastIndexOf(File.separatorChar) + 1, + filePath.lastIndexOf('.') + ); + } + + private String getIndexMapping(String filePath) throws IOException { + Path path = Paths.get(getResourceFilePath(MAPPINGS_FOLDER_ROOT + filePath)); + if (Files.notExists(path)) { + return ""; + } + return new String(Files.readAllBytes(path)); + } + +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/annotation/DocTestConfig.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/annotation/DocTestConfig.java similarity index 84% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/annotation/DocTestConfig.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/annotation/DocTestConfig.java index a08c342526..a44f8471b5 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/annotation/DocTestConfig.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/annotation/DocTestConfig.java @@ -15,12 +15,12 @@ package com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation; -import java.lang.annotation.Retention; -import java.lang.annotation.Target; - import static java.lang.annotation.ElementType.TYPE; import static java.lang.annotation.RetentionPolicy.RUNTIME; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + /** * Configuration to initializing the set up for a doc test. */ @@ -28,16 +28,18 @@ @Target(value = TYPE) public @interface DocTestConfig { - /** - * Path of the template - * @return path - */ - String template(); + /** + * Path of the template + * + * @return path + */ + String template(); - /** - * Path of the test data used. - * @return path - */ - String[] testData() default {}; + /** + * Path of the test data used. + * + * @return path + */ + String[] testData() default {}; } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/annotation/Section.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/annotation/Section.java similarity index 93% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/annotation/Section.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/annotation/Section.java index 18111e9f3f..83b0d12e67 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/annotation/Section.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/annotation/Section.java @@ -15,12 +15,12 @@ package com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation; -import java.lang.annotation.Retention; -import java.lang.annotation.Target; - import static java.lang.annotation.ElementType.METHOD; import static java.lang.annotation.RetentionPolicy.RUNTIME; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + /** * This annotation is used to indicate current method is a valid method for doc generation * and it is supposed to run in the specified order. @@ -29,9 +29,9 @@ @Target(value = METHOD) public @interface Section { - /** - * @return section order - */ - int value() default 0; + /** + * @return section order + */ + int value() default 0; } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/Body.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/Body.java similarity index 64% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/Body.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/Body.java index 502b58186e..273702b8c9 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/Body.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/Body.java @@ -23,18 +23,19 @@ */ class Body { - private final String[] fieldValues; + private final String[] fieldValues; - /** - * Request body built from field value pairs. - * @param fieldValues field and values in "'field": 'value'" format that can assemble to JSON directly - */ - Body(String... fieldValues) { - this.fieldValues = fieldValues; - } + /** + * Request body built from field value pairs. + * + * @param fieldValues field and values in "'field": 'value'" format that can assemble to JSON directly + */ + Body(String... fieldValues) { + this.fieldValues = fieldValues; + } - @Override - public String toString() { - return Arrays.stream(fieldValues).collect(Collectors.joining(",", "{", "}")); - } + @Override + public String toString() { + return Arrays.stream(fieldValues).collect(Collectors.joining(",", "{", "}")); + } } diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/DocBuilder.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/DocBuilder.java new file mode 100644 index 0000000000..9422d6fdaa --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/DocBuilder.java @@ -0,0 +1,234 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.doctest.core.builder; + +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequest.UrlParam; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.KIBANA_REQUEST; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.TABLE_RESPONSE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlAction.EXPLAIN_API_ENDPOINT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; + +import com.amazon.opendistroforelasticsearch.sql.doctest.core.markup.Document; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequest; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; +import com.google.common.base.Strings; +import java.util.Arrays; +import org.elasticsearch.client.RestClient; + +/** + * Build document by custom DSL. To make it more readable, each doc test needs to implement this interface + * and provide things required, such as client connection and document handle. As benefit, they can use the + * DSL to build document in readable and fluent way. + */ +public interface DocBuilder { + + String DOCUMENT_FOLDER_ROOT = "/docs/user/"; + String IMAGE_FOLDER_PATH = DOCUMENT_FOLDER_ROOT + "img/"; + + /** + * Get client connection to cluster for sending request + * + * @return REST client + */ + RestClient restClient(); + + /** + * Open document file to write + * + * @return document class + */ + Document openDocument(); + + default void section(String title, String description, Example... examples) { + section(title, description, new String[0], examples); + } + + /** + * Entry point to start building document by DSL. + * Each section consists of: + * 1. Title + * 2. Description + * 3. Example(s) + * 3.1 Description + * 3.2 [Sample request] + * 3.3 [Explain request] + * 3.4 [Explain output] + * 3.5 [Result set] + * + * @param title title of the section + * @param description description paragraph + * @param examples examples for the section + */ + default void section(String title, String description, String[] images, Example... examples) { + try (Document document = openDocument()) { + document.section(title); + + if (!description.isEmpty()) { + document.subSection("Description").paragraph(description); + } + + if (images.length > 0) { + document.subSection("Syntax"); + for (String image : images) { + // Convert image name ex. "rdd/queryStatement.png" to "queryStatement" as description. + String imageDesc = image.substring(image.lastIndexOf('/') + 1, image.lastIndexOf('.')); + document.image("Rule ``" + imageDesc + "``", IMAGE_FOLDER_PATH + image); + } + } + + for (int i = 0; i < examples.length; i++) { + String exampleTitle; + if (examples.length > 1) { + exampleTitle = "Example " + (i + 1); + } else { + exampleTitle = "Example"; + } + if (!Strings.isNullOrEmpty(examples[i].getTitle())) { + exampleTitle += (": " + examples[i].getTitle()); + } + document.subSection(exampleTitle); + + Example example = examples[i]; + if (!example.getDescription().isEmpty()) { + document.paragraph(example.getDescription()); + } + + document.codeBlock("SQL query", example.getQuery()). + codeBlock("Explain query", example.getExplainQuery()). + codeBlock("Explain", example.getExplainResult()); + + if (example.isTable()) { + document.table("Result set", example.getResult()); + } else { + document.codeBlock("Result set", example.getResult()); + } + } + } + } + + default Example example(String description, Requests requests) { + return example("", description, requests); + } + + /** + * Construct an example by default query and explain format + */ + default Example example(String title, String description, Requests requests) { + return example(title, description, requests, + queryFormat(KIBANA_REQUEST, TABLE_RESPONSE), + explainFormat(IGNORE_REQUEST, PRETTY_JSON_RESPONSE) + ); + } + + default Example example(String description, + Requests requests, + Formats queryFormat, + Formats explainFormat) { + return example("", description, requests, queryFormat, explainFormat); + } + + default Example example(String title, + String description, + Requests requests, + Formats queryFormat, + Formats explainFormat) { + Example example = new Example(); + example.setTitle(title); + example.setDescription(description); + example.setQuery(queryFormat.format(requests.query())); + example.setTable(queryFormat.isTableFormat()); + example.setResult(queryFormat.format(requests.queryResponse())); + example.setExplainQuery(explainFormat.format(requests.explain())); + example.setExplainResult(explainFormat.format(requests.explainResponse())); + return example; + } + + /** + * Simple method just for readability + */ + default String title(String title) { + return title; + } + + default String description(String... sentences) { + return String.join(" ", sentences); + } + + default String[] images(String... images) { + return images; + } + + default Formats queryFormat(SqlRequestFormat requestFormat, SqlResponseFormat responseFormat) { + return new Formats(requestFormat, responseFormat); + } + + default Formats explainFormat(SqlRequestFormat requestFormat, SqlResponseFormat responseFormat) { + return new Formats(requestFormat, responseFormat); + } + + default Requests get(String sql) { + Body body = new Body("\"query\":\"" + sql + "\""); + return new Requests( + restClient(), + new SqlRequest("GET", QUERY_API_ENDPOINT, "", new UrlParam("sql", sql)), + new SqlRequest("POST", EXPLAIN_API_ENDPOINT, body.toString()) + ); + } + + default Requests put(String name, Object value) { + String setting = value == null ? + StringUtils.format("\"%s\": {\"%s\": null}", "transient", name) : + StringUtils.format("\"%s\": {\"%s\": \"%s\"}", "transient", name, value); + return new Requests( + restClient(), + new SqlRequest("PUT", "/_opendistro/_sql/settings", new Body(setting).toString()), + null + ); + } + + default String multiLine(String... lines) { + return String.join("\\n", lines); + } + + /** + * Query by a simple SQL is too common and deserve a dedicated overload method + */ + default Requests post(String sql, UrlParam... params) { + return post(new Body("\"query\":\"" + sql + "\""), params); + } + + default Requests post(Body body, UrlParam... params) { + String bodyStr = body.toString(); + return new Requests( + restClient(), + new SqlRequest("POST", QUERY_API_ENDPOINT, bodyStr, params), + new SqlRequest("POST", EXPLAIN_API_ENDPOINT, bodyStr) + ); + } + + default Body body(String... fieldValues) { + return new Body(fieldValues); + } + + default UrlParam[] params(String... keyValues) { + return Arrays.stream(keyValues).map(UrlParam::new).toArray(UrlParam[]::new); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/Example.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/Example.java new file mode 100644 index 0000000000..01c4370bb6 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/Example.java @@ -0,0 +1,113 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.doctest.core.builder; + +/** + * Example value object. + */ +public class Example { + + /** + * Title for the example section + */ + private String title; + + /** + * Description for the example + */ + private String description; + + /** + * Sample SQL query + */ + private String query; + + /** + * Query result set + */ + private String result; + + /** + * Is result set formatted in table (markup handle table in different way + */ + private boolean isTable; + + /** + * Explain query correspondent to the sample query + */ + private String explainQuery; + + /** + * Result of explain + */ + private String explainResult; + + public String getTitle() { + return title; + } + + public String getDescription() { + return description; + } + + public String getQuery() { + return query; + } + + public String getResult() { + return result; + } + + public boolean isTable() { + return isTable; + } + + public String getExplainQuery() { + return explainQuery; + } + + public String getExplainResult() { + return explainResult; + } + + public void setTitle(String title) { + this.title = title; + } + + public void setDescription(String description) { + this.description = description; + } + + public void setQuery(String query) { + this.query = query; + } + + public void setResult(String result) { + this.result = result; + } + + public void setTable(boolean table) { + isTable = table; + } + + public void setExplainQuery(String explainQuery) { + this.explainQuery = explainQuery; + } + + public void setExplainResult(String explainResult) { + this.explainResult = explainResult; + } +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/Formats.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/Formats.java similarity index 69% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/Formats.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/Formats.java index 14d96be5cb..e1163146bc 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/Formats.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/Formats.java @@ -15,36 +15,36 @@ package com.amazon.opendistroforelasticsearch.sql.doctest.core.builder; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.TABLE_RESPONSE; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.TABLE_UNSORTED_RESPONSE; + import com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequest; import com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat; import com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponse; import com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.TABLE_RESPONSE; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.TABLE_UNSORTED_RESPONSE; - /** * Request and response format tuple. */ class Formats { - private final SqlRequestFormat requestFormat; - private final SqlResponseFormat responseFormat; + private final SqlRequestFormat requestFormat; + private final SqlResponseFormat responseFormat; - Formats(SqlRequestFormat requestFormat, SqlResponseFormat responseFormat) { - this.requestFormat = requestFormat; - this.responseFormat = responseFormat; - } + Formats(SqlRequestFormat requestFormat, SqlResponseFormat responseFormat) { + this.requestFormat = requestFormat; + this.responseFormat = responseFormat; + } - String format(SqlRequest request) { - return requestFormat.format(request); - } + String format(SqlRequest request) { + return requestFormat.format(request); + } - String format(SqlResponse response) { - return responseFormat.format(response); - } + String format(SqlResponse response) { + return responseFormat.format(response); + } - boolean isTableFormat() { - return responseFormat == TABLE_RESPONSE || responseFormat == TABLE_UNSORTED_RESPONSE; - } + boolean isTableFormat() { + return responseFormat == TABLE_RESPONSE || responseFormat == TABLE_UNSORTED_RESPONSE; + } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/ListItems.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/ListItems.java similarity index 62% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/ListItems.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/ListItems.java index 6267a1cd22..3096fa7009 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/ListItems.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/ListItems.java @@ -15,26 +15,26 @@ package com.amazon.opendistroforelasticsearch.sql.doctest.core.builder; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; /** * Item list */ public class ListItems { - private final StringBuilder list = new StringBuilder(); - private int index = 0; + private final StringBuilder list = new StringBuilder(); + private int index = 0; - public void addItem(String text) { - list.append(index()).append(text).append('\n'); - } + public void addItem(String text) { + list.append(index()).append(text).append('\n'); + } - private String index() { - index++; - return StringUtils.format("%d. ", index); - } + private String index() { + index++; + return StringUtils.format("%d. ", index); + } - @Override - public String toString() { - return list.toString(); - } + @Override + public String toString() { + return list.toString(); + } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/Requests.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/Requests.java similarity index 54% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/Requests.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/Requests.java index 9ea876573f..770c609c36 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/Requests.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/Requests.java @@ -17,46 +17,45 @@ import com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequest; import com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponse; -import org.elasticsearch.client.RestClient; - import java.util.Objects; +import org.elasticsearch.client.RestClient; /** * Query and explain request tuple. */ public class Requests { - private final RestClient client; - private final SqlRequest query; - private final SqlRequest explain; + private final RestClient client; + private final SqlRequest query; + private final SqlRequest explain; - public Requests(RestClient client, SqlRequest query) { - this(client, query, SqlRequest.NONE); - } + public Requests(RestClient client, SqlRequest query) { + this(client, query, SqlRequest.NONE); + } - public Requests(RestClient client, SqlRequest query, SqlRequest explain) { - this.client = client; - this.query = query; - this.explain = explain; - } + public Requests(RestClient client, SqlRequest query, SqlRequest explain) { + this.client = client; + this.query = query; + this.explain = explain; + } - public SqlRequest query() { - return query; - } + public SqlRequest query() { + return query; + } - public SqlResponse queryResponse() { - Objects.requireNonNull(query, "Query request is required"); - return query.send(client); - } + public SqlResponse queryResponse() { + Objects.requireNonNull(query, "Query request is required"); + return query.send(client); + } - public SqlRequest explain() { - return explain; - } + public SqlRequest explain() { + return explain; + } - public SqlResponse explainResponse() { - if (explain == SqlRequest.NONE) { - return SqlResponse.NONE; - } - return explain.send(client); + public SqlResponse explainResponse() { + if (explain == SqlRequest.NONE) { + return SqlResponse.NONE; } + return explain.send(client); + } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/markup/Document.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/markup/Document.java similarity index 68% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/markup/Document.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/markup/Document.java index a7f7a0a2c9..bf4696e54d 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/markup/Document.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/markup/Document.java @@ -22,22 +22,22 @@ */ public interface Document extends Closeable { - /** - * Remove checked IOException in method signature. - */ - @Override - void close(); + /** + * Remove checked IOException in method signature. + */ + @Override + void close(); - Document section(String title); + Document section(String title); - Document subSection(String title); + Document subSection(String title); - Document paragraph(String text); + Document paragraph(String text); - Document codeBlock(String description, String code); + Document codeBlock(String description, String code); - Document table(String description, String table); + Document table(String description, String table); - Document image(String description, String filePath); + Document image(String description, String filePath); } diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/markup/RstDocument.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/markup/RstDocument.java new file mode 100644 index 0000000000..7def85dcef --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/markup/RstDocument.java @@ -0,0 +1,110 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.doctest.core.markup; + +import com.google.common.base.Strings; +import java.io.PrintWriter; + +/** + * ReStructure Text document + */ +public class RstDocument implements Document { + + private final PrintWriter docWriter; + + public RstDocument(PrintWriter docWriter) { + this.docWriter = docWriter; + } + + @Override + public Document section(String title) { + return printTitleWithUnderline(title, "="); + } + + @Override + public Document subSection(String title) { + return printTitleWithUnderline(title, "-"); + } + + @Override + public Document paragraph(String text) { + return println(text); + } + + @Override + public Document codeBlock(String description, String code) { + if (!Strings.isNullOrEmpty(code)) { + return println(description + "::", indent(code)); + } + return this; + } + + @Override + public Document table(String description, String table) { + if (!Strings.isNullOrEmpty(table)) { + // RST table is different and not supposed to indent + return println(description + ":", table); + } + return this; + } + + @Override + public Document image(String description, String filePath) { + return println( + description + ":", + ".. image:: " + filePath + ); + } + + @Override + public void close() { + docWriter.close(); + } + + private Document printTitleWithUnderline(String title, String underlineChar) { + return print( + title, + Strings.repeat(underlineChar, title.length()) + ); + } + + /** + * Print each line with a blank line at last + */ + private Document print(String... lines) { + for (String line : lines) { + docWriter.println(line); + } + docWriter.println(); + return this; + } + + /** + * Print each line with a blank line followed + */ + private Document println(String... lines) { + for (String line : lines) { + docWriter.println(line); + docWriter.println(); + } + return this; + } + + private String indent(String text) { + return "\t" + text.replaceAll("\\n", "\n\t"); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/request/SqlRequest.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/request/SqlRequest.java new file mode 100644 index 0000000000..c2bc229d8f --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/request/SqlRequest.java @@ -0,0 +1,107 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.doctest.core.request; + +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.KIBANA_REQUEST; + +import com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponse; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; +import java.io.IOException; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; + +/** + * Request to SQL plugin to isolate Elasticsearch native request + */ +public class SqlRequest { + + public static final SqlRequest NONE = null; + + /** + * Native Elasticsearch request object + */ + private final Request request; + + public SqlRequest(String method, String endpoint, String body, UrlParam... params) { + this.request = makeRequest(method, endpoint, body, params); + } + + /** + * Send request to Elasticsearch via client and create response for it. + * + * @param client restful client connection + * @return sql response + */ + public SqlResponse send(RestClient client) { + try { + return new SqlResponse(client.performRequest(request)); + } catch (IOException e) { + // Some test may expect failure + if (e instanceof ResponseException) { + return new SqlResponse(((ResponseException) e).getResponse()); + } + + throw new IllegalStateException(StringUtils.format( + "Exception occurred during sending request %s", KIBANA_REQUEST.format(this)), e); + } + } + + /** + * Expose request for request formatter. + * + * @return native Elasticsearch format + */ + public Request request() { + return request; + } + + private Request makeRequest(String method, String endpoint, String body, UrlParam[] params) { + Request request = new Request(method, endpoint); + request.setJsonEntity(body); + for (UrlParam param : params) { + request.addParameter(param.key, param.value); + } + + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + request.setOptions(restOptionsBuilder); + return request; + } + + public static class UrlParam { + private String key; + private String value; + + public UrlParam(String key, String value) { + this.key = key; + this.value = value; + } + + public UrlParam(String keyValue) { + int equality = keyValue.indexOf('='); + if (equality == -1) { + throw new IllegalArgumentException(String.format( + "Key value pair is in bad format [%s]", keyValue)); + } + + this.key = keyValue.substring(0, equality); + this.value = keyValue.substring(equality + 1); + } + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/request/SqlRequestFormat.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/request/SqlRequestFormat.java new file mode 100644 index 0000000000..2ef32a53a9 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/request/SqlRequestFormat.java @@ -0,0 +1,134 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.doctest.core.request; + +import static java.util.stream.Collectors.joining; + +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.JsonPrettyFormatter; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; +import com.google.common.base.Charsets; +import com.google.common.io.CharStreams; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.apache.http.Header; +import org.elasticsearch.client.Request; +import org.json.JSONObject; + +/** + * Different SQL request formats. + */ +public enum SqlRequestFormat { + + IGNORE_REQUEST { + @Override + public String format(SqlRequest request) { + return ""; + } + }, + CURL_REQUEST { + @Override + public String format(SqlRequest sqlRequest) { + Request request = sqlRequest.request(); + StringBuilder str = new StringBuilder(); + str.append(">> curl "); + + List
headers = request.getOptions().getHeaders(); + if (!headers.isEmpty()) { + str.append(headers.stream(). + map(header -> StringUtils.format("-H '%s: %s'", header.getName(), header.getValue())). + collect(joining(" ", "", " "))); + } + + str.append(StringUtils.format("-X %s ", request.getMethod())). + append("localhost:9200").append(request.getEndpoint()); + + if (!request.getParameters().isEmpty()) { + str.append(formatParams(request.getParameters())); + } + + String body = body(request); + if (!body.isEmpty()) { + str.append(" -d '"). + append(body). + append('\''); + } + return str.toString(); + } + }, + KIBANA_REQUEST { + @Override + public String format(SqlRequest sqlRequest) { + Request request = sqlRequest.request(); + StringBuilder str = new StringBuilder(); + str.append(request.getMethod()). + append(" "). + append(request.getEndpoint()); + + if (!request.getParameters().isEmpty()) { + str.append(formatParams(request.getParameters())); + } + + str.append('\n'). + append(body(request)); + return str.toString(); + } + }; + + /** + * Format SQL request to specific format for documentation. + * + * @param request sql request + * @return string in specific format + */ + public abstract String format(SqlRequest request); + + @SuppressWarnings("UnstableApiUsage") + protected String body(Request request) { + String body = ""; + try { + InputStream content = request.getEntity().getContent(); + String rawBody = CharStreams.toString(new InputStreamReader(content, Charsets.UTF_8)); + if (!rawBody.isEmpty()) { + JSONObject json = new JSONObject(rawBody); + String sql = json.optString("query"); // '\\n' in literal is replaced by '\n' after unquote + body = JsonPrettyFormatter.format(rawBody); + + // Format and replace multi-line sql literal + if (!sql.isEmpty() && sql.contains("\n")) { + String multiLineSql = + Arrays.stream(sql.split("\\n")). // '\\n' is to escape backslash in regex + collect(joining("\n\t", + "\"\"\"\n\t", + "\n\t\"\"\"")); + body = body.replace("\"" + sql.replace("\n", "\\n") + "\"", multiLineSql); + } + } + } catch (IOException e) { + throw new IllegalStateException("Failed to parse and format body from request", e); + } + return body; + } + + protected String formatParams(Map params) { + return params.entrySet().stream(). + map(e -> e.getKey() + "=" + e.getValue()). + collect(joining("&", "?", "")); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/response/DataTable.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/response/DataTable.java new file mode 100644 index 0000000000..ace101ee3d --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/response/DataTable.java @@ -0,0 +1,92 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.doctest.core.response; + +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; +import com.google.common.base.Strings; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Data table that represent rows of data with a header. + * For now the format is actually in ReST and may need to decouple later. + */ +public class DataTable { + + private final int[] maxWidths; + private final Object[] header; + private final List rows; + + public DataTable(Object[] header) { + this.maxWidths = new int[header.length]; + this.header = header; + this.rows = new ArrayList<>(); + updateMaxWidthForEachColumn(header); + } + + public void addRow(Object[] row) { + rows.add(row); + updateMaxWidthForEachColumn(row); + } + + @Override + public String toString() { + StringBuilder str = new StringBuilder(); + String format = format(); + String separateLine1 = separateLine("-"); + String separateLine2 = separateLine("="); + + str.append(separateLine1). + append('\n'). + append(StringUtils.format(format, header)). + append('\n'). + append(separateLine2). + append('\n'); + + for (Object[] row : rows) { + str.append(StringUtils.format(format, row)). + append('\n'). + append(separateLine("-")). + append('\n'); + } + return str.toString(); + } + + private void updateMaxWidthForEachColumn(Object[] row) { + for (int i = 0; i < row.length; i++) { + maxWidths[i] = Math.max(maxWidths[i], String.valueOf(row[i]).length()); + } + } + + private String separateLine(String separator) { + return Arrays.stream(maxWidths). + mapToObj(width -> Strings.repeat(separator, width)). + collect(Collectors.joining("+", "+", "+")); + } + + /** + * Format as Java String.format needs to make use of auto pad feature. + * For example, to ensure width of 10 and pad spaces, we need to String.format("%10s", str); + */ + private String format() { + return Arrays.stream(maxWidths). + mapToObj(width -> "%" + width + "s"). + collect(Collectors.joining("|", "|", "|")); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/response/SqlResponse.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/response/SqlResponse.java new file mode 100644 index 0000000000..d7fa53ebde --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/response/SqlResponse.java @@ -0,0 +1,79 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.doctest.core.response; + +import com.amazon.opendistroforelasticsearch.sql.util.TestUtils; +import java.io.IOException; +import org.elasticsearch.client.Response; +import org.json.JSONException; +import org.json.JSONObject; + +/** + * Response from SQL plugin + */ +public class SqlResponse { + + public static final SqlResponse NONE = null; + + /** + * Native Elasticsearch response + */ + private final Response response; + + public SqlResponse(Response response) { + this.response = response; + } + + /** + * Parse body in the response. + * + * @return response body + */ + public String body() { + try { + return replaceChangingFields(TestUtils.getResponseBody(response, true)); + } catch (IOException e) { + throw new IllegalStateException("Failed to read response body", e); + } + } + + /** + * In Elasticsearch response, there is field changed between each query, such as "took". + * We have to replace those variants with fake constant to avoid re-generate documents. + * The order of fields in JSON is a little different from original because of internal + * key set in org.json. + */ + private String replaceChangingFields(String response) { + try { + JSONObject root = new JSONObject(response); + if (root.has("took")) { + root.put("took", 100); + } else { + return response; // return original response to minimize impact + } + + if (root.has("_shards")) { + JSONObject shards = root.getJSONObject("_shards"); + shards.put("total", 5); + shards.put("successful", 5); + } + return root.toString(); + } catch (JSONException e) { + // Response is not a valid JSON which is not our interest. + return response; + } + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/response/SqlResponseFormat.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/response/SqlResponseFormat.java new file mode 100644 index 0000000000..2b8d6283d3 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/response/SqlResponseFormat.java @@ -0,0 +1,158 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.doctest.core.response; + +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.JsonPrettyFormatter; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.json.JSONArray; +import org.json.JSONObject; + +/** + * Different SQL response formats + */ +public enum SqlResponseFormat { + + IGNORE_RESPONSE { + @Override + public String format(SqlResponse sqlResponse) { + return ""; + } + }, + ORIGINAL_RESPONSE { + @Override + public String format(SqlResponse sqlResponse) { + return sqlResponse.body(); + } + }, + PRETTY_JSON_RESPONSE { + @Override + public String format(SqlResponse sqlResponse) { + String body = sqlResponse.body(); + try { + return JsonPrettyFormatter.format(body); + } catch (IOException e) { + throw new IllegalStateException( + StringUtils.format("Failed to pretty format response: %s", body), e); + } + } + }, + TABLE_RESPONSE { + @Override + public String format(SqlResponse sqlResponse) { + return format(sqlResponse, true); + } + }, + TABLE_UNSORTED_RESPONSE { + @Override + public String format(SqlResponse sqlResponse) { + return format(sqlResponse, false); + } + }; + + /** + * Format SQL response to specific format for documentation + * + * @param sqlResponse sql response + * @return string in specific format + */ + public abstract String format(SqlResponse sqlResponse); + + /** + * Note that we put this format() here because it's shared by two format enums. + * + * @param sqlResponse original response from plugin + * @param isSorted true to sort the result or just leave it as is + */ + protected String format(SqlResponse sqlResponse, boolean isSorted) { + JSONObject body = new JSONObject(sqlResponse.body()); + if (body.isNull("schema")) { + throw new IllegalStateException( + "Only JDBC response can be formatted to table: " + sqlResponse.body()); + } + + Object[] header = parseHeader(body.getJSONArray("schema")); + List rows = parseDataRows(body.getJSONArray("datarows"), isSorted); + + DataTable table = new DataTable(header); + for (Object[] row : rows) { + table.addRow(row); + } + return table.toString(); + } + + private Object[] parseHeader(JSONArray schema) { + Object[] header = new Object[schema.length()]; + for (int i = 0; i < header.length; i++) { + JSONObject nameType = schema.getJSONObject(i); + header[i] = nameType.optString("alias", nameType.getString("name")); + } + return header; + } + + private List parseDataRows(JSONArray rows, boolean isSorted) { + List rowsToSort = new ArrayList<>(); + for (Object row : rows) { + rowsToSort.add(((JSONArray) row).toList().toArray()); + } + + if (isSorted) { + sort(rowsToSort); + } + return rowsToSort; + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static void sort(List lists) { + lists.sort((list1, list2) -> { + if (list1 == null || list2 == null) { + return compareNullable(list1, list2); + } + + // Assume 2 lists are of same length and all elements are comparable + for (int i = 0; i < list1.length; i++) { + Comparable obj1 = (Comparable) list1[i]; + Comparable obj2 = (Comparable) list2[i]; + + if (obj1 == null || obj2 == null) { + return compareNullable(obj1, obj2); + } + + int result = obj1.compareTo(obj2); + if (result != 0) { + return result; + } + } + return 0; + }); + } + + /** + * Put NULL first (as smaller element) + */ + private static int compareNullable(Object obj1, Object obj2) { + if (obj1 == null && obj2 == null) { + return 0; + } else if (obj1 == null) { + return -1; + } else { // obj2 == null + return 1; + } + } + +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/DataTableTest.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/DataTableTest.java similarity index 66% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/DataTableTest.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/DataTableTest.java index c24e389d4a..5c42dc6556 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/DataTableTest.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/DataTableTest.java @@ -15,49 +15,49 @@ package com.amazon.opendistroforelasticsearch.sql.doctest.core.test; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.response.DataTable; -import org.junit.Test; - import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThat; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.response.DataTable; +import org.junit.Test; + /** * Test cases for {@link DataTable} */ public class DataTableTest { - @Test - public void testSingleColumnTable() { - DataTable table = new DataTable(new Object[]{ "Test Table" }); - table.addRow(new Object[]{ "this is a very long line" }); + @Test + public void testSingleColumnTable() { + DataTable table = new DataTable(new Object[] {"Test Table"}); + table.addRow(new Object[] {"this is a very long line"}); - assertThat( - table.toString(), - is( - "+------------------------+\n" + + assertThat( + table.toString(), + is( + "+------------------------+\n" + "| Test Table|\n" + "+========================+\n" + "|this is a very long line|\n" + "+------------------------+\n" - ) - ); - } - - @Test - public void testTwoColumnsTable() { - DataTable table = new DataTable(new Object[]{ "Test Table", "Very Long Title" }); - table.addRow(new Object[]{ "this is a very long line", "short" }); - - assertThat( - table.toString(), - is( - "+------------------------+---------------+\n" + + ) + ); + } + + @Test + public void testTwoColumnsTable() { + DataTable table = new DataTable(new Object[] {"Test Table", "Very Long Title"}); + table.addRow(new Object[] {"this is a very long line", "short"}); + + assertThat( + table.toString(), + is( + "+------------------------+---------------+\n" + "| Test Table|Very Long Title|\n" + "+========================+===============+\n" + "|this is a very long line| short|\n" + "+------------------------+---------------+\n" - ) - ); - } + ) + ); + } } diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/DocBuilderTest.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/DocBuilderTest.java new file mode 100644 index 0000000000..284717f633 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/DocBuilderTest.java @@ -0,0 +1,238 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.doctest.core.test; + +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.DocBuilder; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.markup.Document; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import org.apache.http.HttpEntity; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +/** + * Test cases for {@link DocBuilder} + */ +@RunWith(MockitoJUnitRunner.class) +public class DocBuilderTest implements DocBuilder { + + private final String queryResponse = + "{\"schema\":[{\"name\":\"firstname\",\"type\":\"text\"}]," + + "\"datarows\":[[\"John\"]],\"total\":10,\"size\":1,\"status\":200}"; + + private final String explainResponse = + "{\"from\":0,\"size\":1,\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]}}"; + + @Mock + private Document document; + + private Verifier verifier; + + @Mock + private RestClient client; + + @Before + public void setUp() throws IOException { + when(document.section(any())).thenReturn(document); + when(document.subSection(any())).thenReturn(document); + when(document.paragraph(any())).thenReturn(document); + when(document.codeBlock(any(), any())).thenReturn(document); + when(document.table(any(), any())).thenReturn(document); + when(document.image(any(), any())).thenReturn(document); + verifier = new Verifier(document); + + when(client.performRequest(any())).then(new Answer() { + private int callCount = 0; + + @Override + public Response answer(InvocationOnMock invocationOnMock) throws IOException { + Response response = mock(Response.class); + HttpEntity entity = mock(HttpEntity.class); + when(response.getEntity()).thenReturn(entity); + + String body = (callCount++ == 0) ? queryResponse : explainResponse; + when(entity.getContent()).thenReturn(new ByteArrayInputStream(body.getBytes())); + return response; + } + }); + } + + @Test + public void sectionShouldIncludeTitleAndDescription() { + section( + title("Test"), + description("This is a test") + ); + + verifier.section("Test"). + subSection("Description"). + paragraph("This is a test"); + } + + @Test + public void sectionShouldIncludeMultiLineSql() { + section( + title("Test"), + description("This is a test"), + example( + description("This is an example for the test"), + post(multiLine( + "SELECT firstname", + "FROM accounts", + "WHERE age > 30") + ) + ) + ); + + verifier.section("Test"). + subSection("Description"). + paragraph("This is a test"). + subSection("Example"). + paragraph("This is an example for the test"). + codeBlock( + "SQL query", + "POST /_opendistro/_sql\n" + + "{\n" + + " \"query\" : \"\"\"\n" + + "\tSELECT firstname\n" + + "\tFROM accounts\n" + + "\tWHERE age > 30\n" + + "\t\"\"\"\n" + + "}" + ); + } + + @Test + public void sectionShouldIncludeExample() { + section( + title("Test"), + description("This is a test"), + images("rdd/querySyntax.png"), + example( + description("This is an example for the test"), + post("SELECT firstname FROM accounts") + ) + ); + + verifier.section("Test"). + subSection("Description"). + paragraph("This is a test"). + image("Rule ``querySyntax``", "/docs/user/img/rdd/querySyntax.png"). + subSection("Example"). + paragraph("This is an example for the test"). + codeBlock( + "SQL query", + "POST /_opendistro/_sql\n" + + "{\n" + + " \"query\" : \"SELECT firstname FROM accounts\"\n" + + "}" + ). + codeBlock( + "Explain", + "{\n" + + " \"from\" : 0,\n" + + " \"size\" : 1,\n" + + " \"_source\" : {\n" + + " \"includes\" : [\n" + + " \"firstname\"\n" + + " ],\n" + + " \"excludes\" : [ ]\n" + + " }\n" + + "}" + ).table( + "Result set", + "+---------+\n" + + "|firstname|\n" + + "+=========+\n" + + "| John|\n" + + "+---------+\n" + ); + } + + @Override + public RestClient restClient() { + return client; + } + + @Override + public Document openDocument() { + return document; + } + + private static class Verifier implements Document { + private final Document mock; + private final InOrder verifier; + + Verifier(Document mock) { + this.mock = mock; + this.verifier = inOrder(mock); + } + + @Override + public void close() { + verifier.verify(mock).close(); + } + + @Override + public Document section(String title) { + verifier.verify(mock).section(title); + return this; + } + + @Override + public Document subSection(String title) { + verifier.verify(mock).subSection(title); + return this; + } + + @Override + public Document paragraph(String text) { + verifier.verify(mock).paragraph(text); + return this; + } + + @Override + public Document codeBlock(String description, String code) { + verifier.verify(mock).codeBlock(description, code); + return this; + } + + @Override + public Document table(String description, String table) { + verifier.verify(mock).table(description, table); + return this; + } + + @Override + public Document image(String description, String filePath) { + verifier.verify(mock).image(description, filePath); + return this; + } + } +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/DocTestTests.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/DocTestTests.java similarity index 100% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/DocTestTests.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/DocTestTests.java diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/RstDocumentTest.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/RstDocumentTest.java similarity index 50% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/RstDocumentTest.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/RstDocumentTest.java index 233aa931e2..adbb4cc724 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/RstDocumentTest.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/RstDocumentTest.java @@ -15,98 +15,97 @@ package com.amazon.opendistroforelasticsearch.sql.doctest.core.test; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.markup.RstDocument; -import org.junit.Before; -import org.junit.Test; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.markup.RstDocument; import java.io.ByteArrayOutputStream; import java.io.PrintWriter; - -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; +import org.junit.Before; +import org.junit.Test; /** * Test cases for {@link RstDocument} */ public class RstDocumentTest { - private ByteArrayOutputStream content; + private ByteArrayOutputStream content; - private RstDocument document; + private RstDocument document; - @Before - public void setUp() { - content = new ByteArrayOutputStream(); - document = new RstDocument(new PrintWriter(content, true)); // Enable auto flush - } + @Before + public void setUp() { + content = new ByteArrayOutputStream(); + document = new RstDocument(new PrintWriter(content, true)); // Enable auto flush + } - @Test - public void testSection() { - document.section("Test Section"); - assertThat( - content.toString(), - is( - "Test Section\n" + + @Test + public void testSection() { + document.section("Test Section"); + assertThat( + content.toString(), + is( + "Test Section\n" + "============\n" + "\n" - ) - ); - } - - @Test - public void testSubSection() { - document.subSection("Test Sub Section"); - assertThat( - content.toString(), - is( - "Test Sub Section\n" + + ) + ); + } + + @Test + public void testSubSection() { + document.subSection("Test Sub Section"); + assertThat( + content.toString(), + is( + "Test Sub Section\n" + "----------------\n" + "\n" - ) - ); - } - - @Test - public void testParagraph() { - document.paragraph("Test paragraph"); - assertThat( - content.toString(), - is( - "Test paragraph\n" + + ) + ); + } + + @Test + public void testParagraph() { + document.paragraph("Test paragraph"); + assertThat( + content.toString(), + is( + "Test paragraph\n" + "\n" - ) - ); - } - - @Test - public void testCodeBlock() { - document.codeBlock("Test code", ">> curl localhost:9200"); - assertThat( - content.toString(), - is( - "Test code::\n" + + ) + ); + } + + @Test + public void testCodeBlock() { + document.codeBlock("Test code", ">> curl localhost:9200"); + assertThat( + content.toString(), + is( + "Test code::\n" + "\n" + "\t>> curl localhost:9200\n" + "\n" - ) - ); - } - - @Test - public void testTable() { - document.table( - "Test table", - "+----------+\n" + + ) + ); + } + + @Test + public void testTable() { + document.table( + "Test table", + "+----------+\n" + "|Test Table|\n" + "+==========+\n" + "| test data|\n" + "+----------+" - ); + ); - assertThat( - content.toString(), - is( - "Test table:\n" + + assertThat( + content.toString(), + is( + "Test table:\n" + "\n" + "+----------+\n" + "|Test Table|\n" + @@ -114,23 +113,23 @@ public void testTable() { "| test data|\n" + "+----------+\n" + "\n" - ) - ); - } - - @Test - public void testImage() { - document.image("Query syntax", "/docs/user/img/query_syntax.png"); - - assertThat( - content.toString(), - is( - "Query syntax:\n" + + ) + ); + } + + @Test + public void testImage() { + document.image("Query syntax", "/docs/user/img/query_syntax.png"); + + assertThat( + content.toString(), + is( + "Query syntax:\n" + "\n" + ".. image:: /docs/user/img/query_syntax.png\n" + "\n" - ) - ); - } + ) + ); + } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/SqlRequestFormatTest.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/SqlRequestFormatTest.java similarity index 62% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/SqlRequestFormatTest.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/SqlRequestFormatTest.java index ddcde16608..1992ebc68c 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/SqlRequestFormatTest.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/SqlRequestFormatTest.java @@ -15,11 +15,6 @@ package com.amazon.opendistroforelasticsearch.sql.doctest.core.test; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequest; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequest.UrlParam; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat; -import org.junit.Test; - import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.CURL_REQUEST; import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.KIBANA_REQUEST; @@ -27,52 +22,57 @@ import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThat; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequest; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequest.UrlParam; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat; +import org.junit.Test; + /** * Test cases for {@link SqlRequestFormat} */ public class SqlRequestFormatTest { - private final SqlRequest sqlRequest = new SqlRequest( - "POST", - "/_opendistro/_sql", - "{\"query\":\"SELECT * FROM accounts\"}", - new UrlParam("format", "jdbc") - ); + private final SqlRequest sqlRequest = new SqlRequest( + "POST", + "/_opendistro/_sql", + "{\"query\":\"SELECT * FROM accounts\"}", + new UrlParam("format", "jdbc") + ); - @Test - public void testIgnoreRequestFormat() { - assertThat(IGNORE_REQUEST.format(sqlRequest), emptyString()); - } + @Test + public void testIgnoreRequestFormat() { + assertThat(IGNORE_REQUEST.format(sqlRequest), emptyString()); + } - @Test - public void testCurlFormat() { - String expected = - ">> curl -H 'Content-Type: application/json' -X POST localhost:9200/_opendistro/_sql?format=jdbc -d '{\n" + + @Test + public void testCurlFormat() { + String expected = + ">> curl -H 'Content-Type: application/json' -X POST localhost:9200/_opendistro/_sql?format=jdbc -d '{\n" + " \"query\" : \"SELECT * FROM accounts\"\n" + "}'"; - assertThat(CURL_REQUEST.format(sqlRequest), is(expected)); - } + assertThat(CURL_REQUEST.format(sqlRequest), is(expected)); + } - @Test - public void testKibanaFormat() { - String expected = - "POST /_opendistro/_sql?format=jdbc\n" + + @Test + public void testKibanaFormat() { + String expected = + "POST /_opendistro/_sql?format=jdbc\n" + "{\n" + " \"query\" : \"SELECT * FROM accounts\"\n" + "}"; - assertThat(KIBANA_REQUEST.format(sqlRequest), is(expected)); - } + assertThat(KIBANA_REQUEST.format(sqlRequest), is(expected)); + } - @Test - public void multiLineSqlInKibanaRequestShouldBeWellFormatted() { - SqlRequest multiLineSqlRequest = new SqlRequest( - "POST", - "/_opendistro/_sql", - "{\"query\":\"SELECT *\\nFROM accounts\\nWHERE age > 30\"}" - ); + @Test + public void multiLineSqlInKibanaRequestShouldBeWellFormatted() { + SqlRequest multiLineSqlRequest = new SqlRequest( + "POST", + "/_opendistro/_sql", + "{\"query\":\"SELECT *\\nFROM accounts\\nWHERE age > 30\"}" + ); - String expected = - "POST /_opendistro/_sql\n" + + String expected = + "POST /_opendistro/_sql\n" + "{\n" + " \"query\" : \"\"\"\n" + "\tSELECT *\n" + @@ -80,7 +80,7 @@ public void multiLineSqlInKibanaRequestShouldBeWellFormatted() { "\tWHERE age > 30\n" + "\t\"\"\"\n" + "}"; - assertThat(KIBANA_REQUEST.format(multiLineSqlRequest), is(expected)); - } + assertThat(KIBANA_REQUEST.format(multiLineSqlRequest), is(expected)); + } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/SqlRequestTest.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/SqlRequestTest.java similarity index 53% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/SqlRequestTest.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/SqlRequestTest.java index 664b01dcbc..3c087b78fb 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/SqlRequestTest.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/SqlRequestTest.java @@ -15,60 +15,59 @@ package com.amazon.opendistroforelasticsearch.sql.doctest.core.test; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + import com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequest; import com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequest.UrlParam; import com.google.common.base.Charsets; import com.google.common.io.CharStreams; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import org.elasticsearch.client.Request; import org.elasticsearch.client.RestClient; import org.junit.Test; import org.mockito.ArgumentCaptor; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; - -import static org.hamcrest.Matchers.hasEntry; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; - /** * Test cases for {@link SqlRequest} */ public class SqlRequestTest { - @Test - public void requestShouldIncludeAllFields() throws IOException { - String method = "POST"; - String endpoint = "/_opendistro/_sql"; - String body = "{\"query\":\"SELECT * FROM accounts\"}"; - String key = "format"; - String value = "jdbc"; - UrlParam param = new UrlParam(key, value); + @Test + public void requestShouldIncludeAllFields() throws IOException { + String method = "POST"; + String endpoint = "/_opendistro/_sql"; + String body = "{\"query\":\"SELECT * FROM accounts\"}"; + String key = "format"; + String value = "jdbc"; + UrlParam param = new UrlParam(key, value); - RestClient client = mock(RestClient.class); - SqlRequest sqlRequest = new SqlRequest(method, endpoint, body, param); - sqlRequest.send(client); + RestClient client = mock(RestClient.class); + SqlRequest sqlRequest = new SqlRequest(method, endpoint, body, param); + sqlRequest.send(client); - ArgumentCaptor argument = ArgumentCaptor.forClass(Request.class); - verify(client).performRequest(argument.capture()); - Request actual = argument.getValue(); - assertThat(actual.getMethod(), is(method)); - assertThat(actual.getEndpoint(), is(endpoint)); - assertThat(actual.getParameters(), hasEntry(key, value)); - assertThat(body(actual), is(body)); - } + ArgumentCaptor argument = ArgumentCaptor.forClass(Request.class); + verify(client).performRequest(argument.capture()); + Request actual = argument.getValue(); + assertThat(actual.getMethod(), is(method)); + assertThat(actual.getEndpoint(), is(endpoint)); + assertThat(actual.getParameters(), hasEntry(key, value)); + assertThat(body(actual), is(body)); + } - @Test(expected = IllegalArgumentException.class) - public void badUrlParamShouldThrowException() { - new UrlParam("test"); - } + @Test(expected = IllegalArgumentException.class) + public void badUrlParamShouldThrowException() { + new UrlParam("test"); + } - private String body(Request request) throws IOException { - InputStream content = request.getEntity().getContent(); - return CharStreams.toString(new InputStreamReader(content, Charsets.UTF_8)); - } + private String body(Request request) throws IOException { + InputStream content = request.getEntity().getContent(); + return CharStreams.toString(new InputStreamReader(content, Charsets.UTF_8)); + } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/SqlResponseFormatTest.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/SqlResponseFormatTest.java similarity index 52% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/SqlResponseFormatTest.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/SqlResponseFormatTest.java index f16025fbd7..a0718ec82e 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/SqlResponseFormatTest.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/SqlResponseFormatTest.java @@ -15,16 +15,6 @@ package com.amazon.opendistroforelasticsearch.sql.doctest.core.test; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponse; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat; -import org.apache.http.HttpEntity; -import org.elasticsearch.client.Response; -import org.junit.Before; -import org.junit.Test; - -import java.io.ByteArrayInputStream; -import java.io.IOException; - import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.ORIGINAL_RESPONSE; import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; @@ -36,43 +26,52 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponse; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import org.apache.http.HttpEntity; +import org.elasticsearch.client.Response; +import org.junit.Before; +import org.junit.Test; + /** * Test cases for {@link SqlResponseFormat} */ public class SqlResponseFormatTest { - private final String expected = - "{" + - "\"schema\":[{\"name\":\"firstname\",\"type\":\"text\"}]," + - "\"datarows\":[[\"John\"]]," + - "\"total\":10," + - "\"size\":1," + - "\"status\":200" + - "}"; - - private SqlResponse sqlResponse; - - @Before - public void setUp() throws IOException { - mockResponse(expected); - } - - @Test - public void testIgnoreResponseFormat() { - assertThat(IGNORE_RESPONSE.format(sqlResponse), emptyString()); - } - - @Test - public void testOriginalFormat() { - assertThat(ORIGINAL_RESPONSE.format(sqlResponse), is(expected + "\n")); - } - - @Test - public void testPrettyJsonFormat() { - assertThat( - PRETTY_JSON_RESPONSE.format(sqlResponse), - is( - "{\n" + + private final String expected = + "{" + + "\"schema\":[{\"name\":\"firstname\",\"type\":\"text\"}]," + + "\"datarows\":[[\"John\"]]," + + "\"total\":10," + + "\"size\":1," + + "\"status\":200" + + "}"; + + private SqlResponse sqlResponse; + + @Before + public void setUp() throws IOException { + mockResponse(expected); + } + + @Test + public void testIgnoreResponseFormat() { + assertThat(IGNORE_RESPONSE.format(sqlResponse), emptyString()); + } + + @Test + public void testOriginalFormat() { + assertThat(ORIGINAL_RESPONSE.format(sqlResponse), is(expected + "\n")); + } + + @Test + public void testPrettyJsonFormat() { + assertThat( + PRETTY_JSON_RESPONSE.format(sqlResponse), + is( + "{\n" + " \"schema\" : [\n" + " {\n" + " \"name\" : \"firstname\",\n" + @@ -88,47 +87,47 @@ public void testPrettyJsonFormat() { " \"size\" : 1,\n" + " \"status\" : 200\n" + "}" - ) - ); - } - - @Test - public void testTableFormat() { - assertThat( - TABLE_RESPONSE.format(sqlResponse), - is( - "+---------+\n" + + ) + ); + } + + @Test + public void testTableFormat() { + assertThat( + TABLE_RESPONSE.format(sqlResponse), + is( + "+---------+\n" + "|firstname|\n" + "+=========+\n" + "| John|\n" + "+---------+\n" - ) - ); - } - - @Test - public void rowsInTableShouldBeSorted() throws IOException { - mockResponse( - "{" + - "\"schema\":[" + - "{\"name\":\"firstname\",\"type\":\"text\"}," + - "{\"name\":\"age\",\"type\":\"integer\"}" + - "]," + - "\"datarows\":[" + - "[\"John\", 30]," + - "[\"John\", 24]," + - "[\"Allen\", 45]" + - "]," + - "\"total\":10," + - "\"size\":3," + - "\"status\":200" + + ) + ); + } + + @Test + public void rowsInTableShouldBeSorted() throws IOException { + mockResponse( + "{" + + "\"schema\":[" + + "{\"name\":\"firstname\",\"type\":\"text\"}," + + "{\"name\":\"age\",\"type\":\"integer\"}" + + "]," + + "\"datarows\":[" + + "[\"John\", 30]," + + "[\"John\", 24]," + + "[\"Allen\", 45]" + + "]," + + "\"total\":10," + + "\"size\":3," + + "\"status\":200" + "}" - ); + ); - assertThat( - TABLE_RESPONSE.format(sqlResponse), - is( - "+---------+---+\n" + + assertThat( + TABLE_RESPONSE.format(sqlResponse), + is( + "+---------+---+\n" + "|firstname|age|\n" + "+=========+===+\n" + "| Allen| 45|\n" + @@ -137,33 +136,33 @@ public void rowsInTableShouldBeSorted() throws IOException { "+---------+---+\n" + "| John| 30|\n" + "+---------+---+\n" - ) - ); - } - - @Test - public void rowsInTableUnsortedShouldMaintainOriginalOrder() throws IOException { - mockResponse( - "{" + - "\"schema\":[" + - "{\"name\":\"firstname\",\"type\":\"text\"}," + - "{\"name\":\"age\",\"type\":\"integer\"}" + - "]," + - "\"datarows\":[" + - "[\"John\", 30]," + - "[\"John\", 24]," + - "[\"Allen\", 45]" + - "]," + - "\"total\":10," + - "\"size\":3," + - "\"status\":200" + + ) + ); + } + + @Test + public void rowsInTableUnsortedShouldMaintainOriginalOrder() throws IOException { + mockResponse( + "{" + + "\"schema\":[" + + "{\"name\":\"firstname\",\"type\":\"text\"}," + + "{\"name\":\"age\",\"type\":\"integer\"}" + + "]," + + "\"datarows\":[" + + "[\"John\", 30]," + + "[\"John\", 24]," + + "[\"Allen\", 45]" + + "]," + + "\"total\":10," + + "\"size\":3," + + "\"status\":200" + "}" - ); + ); - assertThat( - TABLE_UNSORTED_RESPONSE.format(sqlResponse), - is( - "+---------+---+\n" + + assertThat( + TABLE_UNSORTED_RESPONSE.format(sqlResponse), + is( + "+---------+---+\n" + "|firstname|age|\n" + "+=========+===+\n" + "| John| 30|\n" + @@ -172,16 +171,16 @@ public void rowsInTableUnsortedShouldMaintainOriginalOrder() throws IOException "+---------+---+\n" + "| Allen| 45|\n" + "+---------+---+\n" - ) - ); - } - - private void mockResponse(String content) throws IOException { - Response response = mock(Response.class); - HttpEntity entity = mock(HttpEntity.class); - when(response.getEntity()).thenReturn(entity); - when(entity.getContent()).thenReturn(new ByteArrayInputStream(content.getBytes())); - sqlResponse = new SqlResponse(response); - } + ) + ); + } + + private void mockResponse(String content) throws IOException { + Response response = mock(Response.class); + HttpEntity entity = mock(HttpEntity.class); + when(response.getEntity()).thenReturn(entity); + when(entity.getContent()).thenReturn(new ByteArrayInputStream(content.getBytes())); + sqlResponse = new SqlResponse(response); + } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/SqlResponseTest.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/SqlResponseTest.java similarity index 69% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/SqlResponseTest.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/SqlResponseTest.java index d70c3a4a2b..0b597168d5 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/SqlResponseTest.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/SqlResponseTest.java @@ -15,35 +15,34 @@ package com.amazon.opendistroforelasticsearch.sql.doctest.core.test; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponse; -import org.apache.http.HttpEntity; -import org.elasticsearch.client.Response; -import org.junit.Test; - -import java.io.ByteArrayInputStream; -import java.io.IOException; - import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponse; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import org.apache.http.HttpEntity; +import org.elasticsearch.client.Response; +import org.junit.Test; + /** * Test cases for {@link SqlResponse} */ public class SqlResponseTest { - @Test - public void responseBodyShouldRetainNewLine() throws IOException { - Response response = mock(Response.class); - HttpEntity entity = mock(HttpEntity.class); - String expected = "123\nabc\n"; - when(response.getEntity()).thenReturn(entity); - when(entity.getContent()).thenReturn(new ByteArrayInputStream(expected.getBytes())); - - SqlResponse sqlResponse = new SqlResponse(response); - String actual = sqlResponse.body(); - assertThat(actual, is(expected)); - } + @Test + public void responseBodyShouldRetainNewLine() throws IOException { + Response response = mock(Response.class); + HttpEntity entity = mock(HttpEntity.class); + String expected = "123\nabc\n"; + when(response.getEntity()).thenReturn(entity); + when(entity.getContent()).thenReturn(new ByteArrayInputStream(expected.getBytes())); + + SqlResponse sqlResponse = new SqlResponse(response); + String actual = sqlResponse.body(); + assertThat(actual, is(expected)); + } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dml/DeleteIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dml/DeleteIT.java similarity index 62% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dml/DeleteIT.java rename to integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dml/DeleteIT.java index 8f7ef547b3..8f88c16a5d 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dml/DeleteIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dml/DeleteIT.java @@ -15,39 +15,39 @@ package com.amazon.opendistroforelasticsearch.sql.doctest.dml; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.DocTest; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.DocTestConfig; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; - import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.KIBANA_REQUEST; import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.DocTest; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.DocTestConfig; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; + @DocTestConfig(template = "dml/delete.rst", testData = {"accounts.json"}) public class DeleteIT extends DocTest { - @Section(1) - public void delete() { - section( - title("DELETE"), + @Section(1) + public void delete() { + section( + title("DELETE"), + description( + "``DELETE`` statement deletes documents that satisfy the predicates in ``WHERE`` clause.", + "Note that all documents are deleted in the case of ``WHERE`` clause absent." + ), + images("rdd/singleDeleteStatement.png"), + example( description( - "``DELETE`` statement deletes documents that satisfy the predicates in ``WHERE`` clause.", - "Note that all documents are deleted in the case of ``WHERE`` clause absent." + "The ``datarows`` field in this case shows rows impacted, in other words how many", + "documents were just deleted." ), - images("rdd/singleDeleteStatement.png"), - example( - description( - "The ``datarows`` field in this case shows rows impacted, in other words how many", - "documents were just deleted." - ), - post(multiLine( - "DELETE FROM accounts", - "WHERE age > 30" - )), - queryFormat(KIBANA_REQUEST, PRETTY_JSON_RESPONSE), - explainFormat(IGNORE_REQUEST, PRETTY_JSON_RESPONSE) - ) - ); - } + post(multiLine( + "DELETE FROM accounts", + "WHERE age > 30" + )), + queryFormat(KIBANA_REQUEST, PRETTY_JSON_RESPONSE), + explainFormat(IGNORE_REQUEST, PRETTY_JSON_RESPONSE) + ) + ); + } } diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/BasicQueryIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/BasicQueryIT.java new file mode 100644 index 0000000000..cc164de12a --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/BasicQueryIT.java @@ -0,0 +1,319 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.doctest.dql; + +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.KIBANA_REQUEST; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.TABLE_UNSORTED_RESPONSE; + +import com.amazon.opendistroforelasticsearch.sql.doctest.core.DocTest; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.DocTestConfig; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.Example; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.Requests; + +/** + * Doc test for basic SELECT query. + */ +@DocTestConfig(template = "dql/basics.rst", testData = {"accounts.json"}) +public class BasicQueryIT extends DocTest { + + @Section(1) + public void select() { + section( + title("SELECT"), + description( + "``SELECT`` clause specifies which fields in Elasticsearch index should be retrieved."), + images("rdd/selectElements.png", "rdd/selectElement.png"), + example( + title("Selecting All Fields"), + description( + "You can use ``*`` to fetch all fields in the index which is very convenient when you", + "just want to have a quick look at your data." + ), + post("SELECT * FROM accounts") + ), + example( + title("Selecting Specific Fields"), + description( + "More often you would give specific field name(s) in ``SELECT`` clause to", + "avoid large and unnecessary data retrieved." + ), + post("SELECT firstname, lastname FROM accounts") + ), + example( + title("Using Field Alias"), + description( + "Alias is often used to make your query more readable by giving your field a shorter name." + ), + post("SELECT account_number AS num FROM accounts") + ), + example( + title("Selecting Distinct Fields"), + description( + "``DISTINCT`` is useful when you want to de-duplicate and get unique field value.", + "You can provide one or more field names." + ), + post("SELECT DISTINCT age FROM accounts") + ) + ); + } + + @Section(2) + public void from() { + section( + title("FROM"), + description( + "``FROM`` clause specifies Elasticsearch index where the data should be retrieved from.", + "You've seen how to specify a single index in FROM clause in last section. Here we", + "provide examples for more use cases.\n\n" + + "Subquery in ``FROM`` clause is also supported. Please check out the documentation for more details." + ), + images("rdd/tableName.png"), + kibanaExample( + title("Using Index Alias"), + description( + "Similarly you can give index in ``FROM`` clause an alias and use it across clauses in query." + ), + post("SELECT acc.account_number FROM accounts acc") + ), + kibanaExample( + title("Selecting From Multiple Indices by Index Pattern"), + description( + "Alternatively you can query from multiple indices of similar names by index pattern.", + "This is very convenient for indices created by Logstash index template with date as suffix." + ), + post("SELECT account_number FROM account*") + ), + kibanaExample( + title("[Deprecating] Selecting From Specific Index Type"), + description( + "You can also specify type name explicitly though this has been deprecated in", + "later Elasticsearch version." + ), + post("SELECT account_number FROM accounts/account") + ) + ); + } + + @Section(3) + public void where() { + section( + title("WHERE"), + description( + "``WHERE`` clause specifies only Elasticsearch documents that meet the criteria should be affected.", + "It consists of predicates that uses ``=``, ``<>``, ``>``, ``>=``, ``<``, ``<=``, ``IN``,", + "``BETWEEN``, ``LIKE``, ``IS NULL`` or ``IS NOT NULL``. These predicates can be combined by", + "logical operator ``NOT``, ``AND`` or ``OR`` to build more complex expression.\n\n" + + "For ``LIKE`` and other full text search topics, please refer to Full Text Search documentation.\n\n" + + "Besides SQL query, WHERE clause can also be used in SQL statement such as ``DELETE``. Please refer to", + "Data Manipulation Language documentation for details." + ), + example( + title("Comparison Operators"), + description( + "Basic comparison operators, such as ``=``, ``<>``, ``>``, ``>=``, ``<``, ``<=``, can work for", + "number, string or date.", + "``IN`` and ``BETWEEN`` is convenient for comparison with multiple values or a range." + ), + post(multiLine( + "SELECT account_number", + "FROM accounts", + "WHERE account_number = 1" + )) + ), + example( + title("Missing Fields"), + description( + "As NoSQL database, Elasticsearch allows for flexible schema that documents in an index may have", + "different fields. In this case, you can use ``IS NULL`` or ``IS NOT NULL`` to retrieve missing", + "fields or existing fields only.\n\n" + + "Note that for now we don't differentiate missing field and field set to ``NULL`` explicitly." + ), + post(multiLine( + "SELECT account_number, employer", + "FROM accounts", + "WHERE employer IS NULL" + )) + ) + ); + } + + @Section(4) + public void groupBy() { + section( + title("GROUP BY"), + description( + "``GROUP BY`` groups documents with same field value into buckets. It is often used along with", + "aggregation functions to aggregate inside each bucket. Please refer to SQL Functions documentation", + "for more details.\n\n" + + "Note that ``WHERE`` clause is applied before ``GROUP BY`` clause." + ), + example( + title("Grouping by Fields"), + description(), + post(multiLine( + "SELECT age", + "FROM accounts", + "GROUP BY age" + )) + ), + example( + title("Grouping by Field Alias"), + description("Field alias is accessible in ``GROUP BY`` clause."), + post(multiLine( + "SELECT account_number AS num", + "FROM accounts", + "GROUP BY num" + )) + ), + example( + title("Grouping by Ordinal"), + description( + "Alternatively field ordinal in ``SELECT`` clause can be used too. However this is not", + "recommended because your ``GROUP BY`` clause depends on fields in ``SELECT`` clause", + "and require to change accordingly." + ), + post(multiLine( + "SELECT age", + "FROM accounts", + "GROUP BY 1" + )) + ), + example( + title("Grouping by Scalar Function"), + description( + "Scalar function can be used in ``GROUP BY`` clause and it's required to be present in", + "``SELECT`` clause too." + ), + post(multiLine( + "SELECT ABS(age) AS a", + "FROM accounts", + "GROUP BY ABS(age)" + )) + ) + ); + } + + @Section(5) + public void having() { + section( + title("HAVING"), + description( + "``HAVING`` clause filters result from ``GROUP BY`` clause by predicate(s). Because of this,", + "aggregation function, even different from those on ``SELECT`` clause, can be used in predicate." + ), + example( + description(), + post(multiLine( + "SELECT age, MAX(balance)", + "FROM accounts", + "GROUP BY age", + "HAVING MIN(balance) > 10000" + )) + ) + ); + } + + @Section(6) + public void orderBy() { + section( + title("ORDER BY"), + description( + "``ORDER BY`` clause specifies which fields used to sort the result and in which direction."), + orderByExample( + title("Ordering by Fields"), + description( + "Besides regular field names, ordinal, alias or scalar function can also be used similarly", + "as in ``GROUP BY``. ``ASC`` (by default) or ``DESC`` can be appended to indicate sorting in", + "ascending or descending order." + ), + post("SELECT account_number FROM accounts ORDER BY account_number DESC") + ), + orderByExample( + title("Specifying Order for Null"), + description( + "Additionally you can specify if documents with missing field be put first or last.", + "The default behavior of Elasticsearch is to return nulls or missing last.", + "You can make them present before non-nulls by using ``IS NOT NULL``." + ), + post(multiLine( + "SELECT employer", + "FROM accounts", + "ORDER BY employer IS NOT NULL" + )) + ) + ); + } + + @Section(7) + public void limit() { + section( + title("LIMIT"), + description( + "Mostly specifying maximum number of documents returned is necessary to prevent fetching", + "large amount of data into memory. `LIMIT` clause is helpful in this case." + ), + example( + title("Limiting Result Size"), + description( + "Given a positive number, ``LIMIT`` uses it as page size to fetch result of that size at most." + ), + post(multiLine( + "SELECT account_number", + "FROM accounts", + "ORDER BY account_number LIMIT 1" + )) + ), + example( + title("Fetching at Offset"), + description( + "Offset position can be given as first argument to indicate where to start fetching.", + "This can be used as simple pagination solution though it's inefficient on large index.", + "Generally ``ORDER BY`` is required in this case to ensure the same order between pages." + ), + post(multiLine( + "SELECT account_number", + "FROM accounts", + "ORDER BY account_number LIMIT 1, 1" + )) + ) + ); + } + + /** + * Document only Kibana request for example and ignore response as well as explain + */ + private Example kibanaExample(String title, String description, Requests requests) { + return example(title, description, requests, + queryFormat(KIBANA_REQUEST, IGNORE_RESPONSE), + explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) + ); + } + + /** + * Example for ORDER BY needs to maintain the order in original result + */ + private Example orderByExample(String title, String description, Requests requests) { + return example(title, description, requests, + queryFormat(KIBANA_REQUEST, TABLE_UNSORTED_RESPONSE), + explainFormat(IGNORE_REQUEST, PRETTY_JSON_RESPONSE) + ); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/ComplexQueryIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/ComplexQueryIT.java new file mode 100644 index 0000000000..4592c624e3 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/ComplexQueryIT.java @@ -0,0 +1,201 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.doctest.dql; + +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.KIBANA_REQUEST; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.TABLE_RESPONSE; + +import com.amazon.opendistroforelasticsearch.sql.doctest.core.DocTest; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.DocTestConfig; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.Example; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.Requests; +import org.junit.Ignore; + +@DocTestConfig(template = "dql/complex.rst", testData = {"accounts.json", "employees_nested.json"}) +public class ComplexQueryIT extends DocTest { + + @Section(1) + public void subquery() { + section( + title("Subquery"), + description( + "A subquery is a complete ``SELECT`` statement which is used within another statement", + "and enclosed in parenthesis. From the explain output, you can notice that some subquery", + "are actually transformed to an equivalent join query to execute." + ), + /* + example( + title("Scalar Value Subquery"), + description( + "" + ), + post( + "SELECT firstname, lastname, balance " + + "FROM accounts " + + "WHERE balance >= ( " + + " SELECT AVG(balance) FROM accounts " + + ") " + ) + ),*/ + example( + title("Table Subquery"), + description(""), + post(multiLine( + "SELECT a1.firstname, a1.lastname, a1.balance", + "FROM accounts a1", + "WHERE a1.account_number IN (", + " SELECT a2.account_number", + " FROM accounts a2", + " WHERE a2.balance > 10000", + ")" + )) + ), + example( + title("Subquery in FROM Clause"), + description(""), + post(multiLine( + "SELECT a.f, a.l, a.a", + "FROM (", + " SELECT firstname AS f, lastname AS l, age AS a", + " FROM accounts", + " WHERE age > 30", + ") AS a" + )) + ) + ); + } + + @Section(2) + public void joins() { + section( + title("JOINs"), + description( + "A ``JOIN`` clause combines columns from one or more indices by using values common to each." + ), + images("rdd/tableSource.png", "rdd/joinPart.png"), + example( + title("Inner Join"), + description( + "Inner join is very commonly used that creates a new result set by combining columns", + "of two indices based on the join predicates specified. It iterates both indices and", + "compare each document to find all that satisfy the join predicates. Keyword ``JOIN``", + "is used and preceded by ``INNER`` keyword optionally. The join predicate(s) is specified", + "by ``ON`` clause.\n\n", + "Remark that the explain API output for join queries looks complicated. This is because", + "a join query is associated with two Elasticsearch DSL queries underlying and execute in", + "the separate query planner framework. You can interpret it by looking into the logical", + "plan and physical plan." + ), + post(multiLine( + "SELECT", + " a.account_number, a.firstname, a.lastname,", + " e.id, e.name", + "FROM accounts a", + "JOIN employees_nested e", + " ON a.account_number = e.id" + )) + ), + joinExampleWithoutExplain( + title("Cross Join"), + description( + "Cross join or Cartesian join combines each document from the first index with each from", + "the second. The result set is the Cartesian Product of documents from both indices.", + "It appears to be similar to inner join without ``ON`` clause to specify join condition.\n\n", + "Caveat: It is risky to do cross join even on two indices of medium size. This may trigger", + "our circuit breaker to terminate the query to avoid out of memory issue." + ), + post(multiLine( + "SELECT", + " a.account_number, a.firstname, a.lastname,", + " e.id, e.name", + "FROM accounts a", + "JOIN employees_nested e" + )) + ), + joinExampleWithoutExplain( + title("Outer Join"), + description( + "Outer join is used to retain documents from one or both indices although it does not satisfy", + "join predicate. For now, only ``LEFT OUTER JOIN`` is supported to retain rows from first index.", + "Note that keyword ``OUTER`` is optional." + ), + post(multiLine( + "SELECT", + " a.account_number, a.firstname, a.lastname,", + " e.id, e.name", + "FROM accounts a", + "LEFT JOIN employees_nested e", + " ON a.account_number = e.id" + )) + ) + ); + } + + @Ignore("Multi-query doesn't work for default format: https://github.com/opendistro-for-elasticsearch/sql/issues/388") + @Section(3) + public void setOperations() { + section( + title("Set Operations"), + description( + "Set operations allow results of multiple queries to be combined into a single result set.", + "The results to be combined are required to be of same type. In other word, they require to", + "have same column. Otherwise, a semantic analysis exception is raised." + ), + example( + title("UNION Operator"), + description( + "A ``UNION`` clause combines the results of two queries into a single result set. Duplicate rows", + "are removed unless ``UNION ALL`` clause is being used. A common use case of ``UNION`` is to combine", + "result set from data partitioned in indices daily or monthly." + ), + post(multiLine( + "SELECT balance, firstname, lastname", + "FROM accounts WHERE balance < 10000", + "UNION", + "SELECT balance, firstname, lastname", + "FROM accounts WHERE balance > 30000" + )) + ), + example( + title("MINUS Operator"), + description( + "A ``MINUS`` clause takes two queries too but returns resulting rows of first query that", + "do not appear in the other query. Duplicate rows are removed automatically as well." + ), + post(multiLine( + "SELECT balance, age", + "FROM accounts", + "WHERE balance < 10000", + "MINUS", + "SELECT balance, age", + "FROM accounts", + "WHERE age < 35" + )) + ) + ); + } + + private Example joinExampleWithoutExplain(String title, String description, Requests requests) { + return example(title, description, requests, + queryFormat(KIBANA_REQUEST, TABLE_RESPONSE), + explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) + ); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/MetaDataQueryIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/MetaDataQueryIT.java new file mode 100644 index 0000000000..a63e7285cc --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/MetaDataQueryIT.java @@ -0,0 +1,75 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.doctest.dql; + +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.KIBANA_REQUEST; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.TABLE_RESPONSE; + +import com.amazon.opendistroforelasticsearch.sql.doctest.core.DocTest; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.DocTestConfig; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.Example; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.Requests; + +@DocTestConfig(template = "dql/metadata.rst", testData = {"accounts.json", "employees_nested.json"}) +public class MetaDataQueryIT extends DocTest { + + @Section(1) + public void queryMetaData() { + section( + title("Querying Metadata"), + description( + "You can query your indices metadata by ``SHOW`` and ``DESCRIBE`` statement. These commands are", + "very useful for database management tool to enumerate all existing indices and get basic information", + "from the cluster." + ), + images("rdd/showStatement.png", "rdd/showFilter.png"), + metadataQueryExample( + title("Show All Indices Information"), + description( + "``SHOW`` statement lists all indices that match the search pattern. By using wildcard '%',", + "information for all indices in the cluster is returned." + ), + post("SHOW TABLES LIKE %") + ), + metadataQueryExample( + title("Show Specific Index Information"), + description( + "Here is an example that searches metadata for index name prefixed by 'acc'"), + post("SHOW TABLES LIKE acc%") + ), + metadataQueryExample( + title("Describe Index Fields Information"), + description( + "``DESCRIBE`` statement lists all fields for indices that can match the search pattern."), + post("DESCRIBE TABLES LIKE accounts") + ) + ); + } + + /** + * Explain doesn't work for SHOW/DESCRIBE so skip it + */ + private Example metadataQueryExample(String title, String description, Requests requests) { + return example(title, description, requests, + queryFormat(KIBANA_REQUEST, TABLE_RESPONSE), + explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) + ); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/SQLFunctionsIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/SQLFunctionsIT.java new file mode 100644 index 0000000000..3f724f153a --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/SQLFunctionsIT.java @@ -0,0 +1,57 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.doctest.dql; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.TypeExpression.TypeExpressionSpec; + +import com.amazon.opendistroforelasticsearch.sql.doctest.core.DocTest; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.DocTestConfig; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.function.ScalarFunction; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; + +@DocTestConfig(template = "dql/functions.rst") +public class SQLFunctionsIT extends DocTest { + + /** + * List only specifications of all SQL functions supported for now + */ + @Section + public void listFunctions() { + for (ScalarFunction func : ScalarFunction + .values()) { // Java Enum.values() return enums in order they are defined + section( + title(func.getName()), + description(listFunctionSpecs(func)) + ); + } + } + + private String listFunctionSpecs(ScalarFunction func) { + TypeExpressionSpec[] specs = func.specifications(); + if (specs.length == 0) { + return "Specification is undefined and type check is skipped for now"; + } + + StringBuilder specStr = new StringBuilder("Specifications: \n\n"); + for (int i = 0; i < specs.length; i++) { + specStr.append( + StringUtils.format("%d. %s%s\n", (i + 1), func.getName(), specs[i]) + ); + } + return specStr.toString(); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/interfaces/EndpointIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/interfaces/EndpointIT.java new file mode 100644 index 0000000000..100ff9afd0 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/interfaces/EndpointIT.java @@ -0,0 +1,84 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.doctest.interfaces; + +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.CURL_REQUEST; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; + +import com.amazon.opendistroforelasticsearch.sql.doctest.core.DocTest; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.DocTestConfig; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; + +/** + * Doc test for endpoints to access the plugin. + */ +@DocTestConfig(template = "interfaces/endpoint.rst", testData = {"accounts.json"}) +public class EndpointIT extends DocTest { + + @Section(1) + public void queryByPost() { + section( + title("POST"), + description("You can also send HTTP POST request with your query in request body."), + example( + description(), + post("SELECT * FROM accounts"), + queryFormat(CURL_REQUEST, IGNORE_RESPONSE), + explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) + ) + ); + } + + @Section(2) + public void explainQuery() { + section( + title("Explain"), + description( + "To translate your query, send it to explain endpoint. The explain output is Elasticsearch", + "domain specific language (DSL) in JSON format. You can just copy and paste it to your", + "console to run it against Elasticsearch directly." + ), + example( + description(), + post("SELECT firstname, lastname FROM accounts WHERE age > 20"), + queryFormat(IGNORE_REQUEST, IGNORE_RESPONSE), + explainFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE) + ) + ); + } + + @Section(3) + public void cursorQuery() { + section( + title("Cursor"), + description( + "To get paginated response for a query, user needs to provide `fetch_size` parameter as part of normal query.", + "The value of `fetch_size` should be greater than `0`. In absence of `fetch_size`, default value of 1000 is used.", + "A value of `0` will fallback to non-paginated response.", + "This feature is only available over `jdbc` format for now." + ), + example( + description(), + post("SELECT firstname, lastname FROM accounts WHERE age > 20 ORDER BY state ASC"), + queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), + explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) + ) + ); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/interfaces/ProtocolIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/interfaces/ProtocolIT.java new file mode 100644 index 0000000000..38b69ef7c2 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/interfaces/ProtocolIT.java @@ -0,0 +1,144 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.doctest.interfaces; + +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.CURL_REQUEST; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.ORIGINAL_RESPONSE; +import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; + +import com.amazon.opendistroforelasticsearch.sql.doctest.core.DocTest; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.DocTestConfig; +import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; + +/** + * Doc test for plugin supported protocols. + */ +@DocTestConfig(template = "interfaces/protocol.rst", testData = {"accounts.json"}) +public class ProtocolIT extends DocTest { + + @Section(1) + public void requestFormat() { + section( + title("Request Format"), + description( + "The body of HTTP POST request can take a few more other fields with SQL query."), + example( + description( + "Use `filter` to add more conditions to Elasticsearch DSL directly." + ), + post( + body( + "\"query\": \"SELECT firstname, lastname, balance FROM accounts\"", + "\"filter\":{\"range\":{\"balance\":{\"lt\":10000}}}" + ) + ), + queryFormat(CURL_REQUEST, IGNORE_RESPONSE), + explainFormat(IGNORE_REQUEST, PRETTY_JSON_RESPONSE) + ), + example( + description("Use `parameters` for actual parameter value in prepared SQL query."), + post( + body( + "\"query\": \"SELECT * FROM accounts WHERE age = ?\"", + "\"parameters\": [{\"type\": \"integer\", \"value\": 30}]" + ) + ), + queryFormat(CURL_REQUEST, IGNORE_RESPONSE), + explainFormat(IGNORE_REQUEST, PRETTY_JSON_RESPONSE) + ) + ); + } + + @Section(2) + public void responseInJDBCFormat() { + section( + title("JDBC Format"), + description( + "By default the plugin return JDBC format. JDBC format is provided for JDBC driver and client side that needs both schema and", + "result set well formatted." + ), + example( + description( + "Here is an example for normal response. The `schema` includes field name and its type", + "and `datarows` includes the result set." + ), + post("SELECT firstname, lastname, age FROM accounts ORDER BY age LIMIT 2"), + queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), + explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) + ), + example( + description( + "If any error occurred, error message and the cause will be returned instead."), + post("SELECT unknown FROM accounts", params("format=jdbc")), + queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), + explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) + ) + ); + } + + @Section(3) + public void originalDSLResponse() { + section( + title("Elasticsearch DSL"), + description( + "The plugin returns original response from Elasticsearch in JSON. Because this is", + "the native response from Elasticsearch, extra efforts are needed to parse and interpret it." + ), + example( + description(), + post("SELECT firstname, lastname, age FROM accounts ORDER BY age LIMIT 2", + params("format=json")), + queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), + explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) + ) + ); + } + + @Section(4) + public void responseInCSVFormat() { + section( + title("CSV Format"), + description("You can also use CSV format to download result set as CSV."), + example( + description(), + post("SELECT firstname, lastname, age FROM accounts ORDER BY age", + params("format=csv")), + queryFormat(CURL_REQUEST, ORIGINAL_RESPONSE), + explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) + ) + ); + } + + @Section(5) + public void responseInRawFormat() { + section( + title("Raw Format"), + description( + "Additionally raw format can be used to pipe the result to other command line tool for post processing." + ), + example( + description(), + post("SELECT firstname, lastname, age FROM accounts ORDER BY age", + params("format=raw")), + queryFormat(CURL_REQUEST, ORIGINAL_RESPONSE), + explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) + ) + ); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/AggregationExpressionIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/AggregationExpressionIT.java new file mode 100644 index 0000000000..5f6310fa7f --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/AggregationExpressionIT.java @@ -0,0 +1,258 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.rows; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.schema; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyDataRows; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifySchema; + +import org.json.JSONObject; +import org.junit.Ignore; +import org.junit.Test; + +public class AggregationExpressionIT extends SQLIntegTestCase { + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + loadIndex(Index.BANK); + } + + @Test + public void noGroupKeySingleFuncOverAggWithoutAliasShouldPass() { + JSONObject response = executeJdbcRequest(String.format( + "SELECT abs(MAX(age)) " + + "FROM %s", + Index.ACCOUNT.getName())); + + verifySchema(response, schema("abs(MAX(age))", null, "long")); + verifyDataRows(response, rows(40)); + } + + @Test + public void noGroupKeyMaxAddMinShouldPass() { + JSONObject response = executeJdbcRequest(String.format( + "SELECT MAX(age) + MIN(age) as add " + + "FROM %s", + Index.ACCOUNT.getName())); + + verifySchema(response, schema("add", "add", "long")); + verifyDataRows(response, rows(60)); + } + + // todo age field should has long type instead of integer type. + @Ignore + @Test + public void noGroupKeyMaxAddLiteralShouldPass() { + JSONObject response = executeJdbcRequest(String.format( + "SELECT MAX(age) + 1 as add " + + "FROM %s", + Index.ACCOUNT.getName())); + + verifySchema(response, schema("add", "add", "long")); + verifyDataRows(response, rows(41)); + } + + @Test + public void noGroupKeyAvgOnIntegerShouldPass() { + JSONObject response = executeJdbcRequest(String.format( + "SELECT AVG(age) as avg " + + "FROM %s", + Index.BANK.getName())); + + verifySchema(response, schema("avg", "avg", "double")); + verifyDataRows(response, rows(34)); + } + + @Test + public void hasGroupKeyAvgOnIntegerShouldPass() { + JSONObject response = executeJdbcRequest(String.format( + "SELECT gender, AVG(age) as avg " + + "FROM %s " + + "GROUP BY gender", + Index.BANK.getName())); + + verifySchema(response, + schema("gender", null, "text"), + schema("avg", "avg", "double")); + verifyDataRows(response, + rows("m", 34.25), + rows("f", 33.666666666666664d)); + } + + @Test + public void hasGroupKeyMaxAddMinShouldPass() { + JSONObject response = executeJdbcRequest(String.format( + "SELECT gender, MAX(age) + MIN(age) as add " + + "FROM %s " + + "GROUP BY gender", + Index.ACCOUNT.getName())); + + verifySchema(response, + schema("gender", null, "text"), + schema("add", "add", "long")); + verifyDataRows(response, + rows("m", 60), + rows("f", 60)); + } + + // todo age field should has long type instead of integer type. + @Ignore + @Test + public void hasGroupKeyMaxAddLiteralShouldPass() { + JSONObject response = executeJdbcRequest(String.format( + "SELECT gender, MAX(age) + 1 as add " + + "FROM %s " + + "GROUP BY gender", + Index.ACCOUNT.getName())); + + verifySchema(response, + schema("gender", null, "text"), + schema("add", "add", "long")); + verifyDataRows(response, + rows("m", 1), + rows("f", 1)); + } + + @Test + public void noGroupKeyLogMaxAddMinShouldPass() { + JSONObject response = executeJdbcRequest(String.format( + "SELECT Log(MAX(age) + MIN(age)) as log " + + "FROM %s", + Index.ACCOUNT.getName())); + + verifySchema(response, schema("log", "log", "double")); + verifyDataRows(response, rows(4.0943445622221d)); + } + + @Test + public void hasGroupKeyLogMaxAddMinShouldPass() { + JSONObject response = executeJdbcRequest(String.format( + "SELECT gender, Log(MAX(age) + MIN(age)) as log " + + "FROM %s " + + "GROUP BY gender", + Index.ACCOUNT.getName())); + + verifySchema(response, + schema("gender", null, "text"), + schema("log", "log", "double")); + verifyDataRows(response, + rows("m", 4.0943445622221d), + rows("f", 4.0943445622221d)); + } + + // todo age field should has long type instead of integer type. + @Ignore + @Test + public void AddLiteralOnGroupKeyShouldPass() { + JSONObject response = executeJdbcRequest(String.format( + "SELECT gender, age+10, max(balance) as max " + + "FROM %s " + + "WHERE gender = 'm' and age < 22 " + + "GROUP BY gender, age " + + "ORDER BY age", + Index.ACCOUNT.getName())); + + verifySchema(response, + schema("gender", null, "text"), + schema("age", "age", "long"), + schema("max", "max", "long")); + verifyDataRows(response, + rows("m", 30, 49568), + rows("m", 31, 49433)); + } + + @Test + public void logWithAddLiteralOnGroupKeyShouldPass() { + JSONObject response = executeJdbcRequest(String.format( + "SELECT gender, Log(age+10) as logAge, max(balance) as max " + + "FROM %s " + + "WHERE gender = 'm' and age < 22 " + + "GROUP BY gender, age " + + "ORDER BY age", + Index.ACCOUNT.getName())); + + verifySchema(response, + schema("gender", null, "text"), + schema("logAge", "logAge", "double"), + schema("max", "max", "long")); + verifyDataRows(response, + rows("m", 3.4011973816621555d, 49568), + rows("m", 3.4339872044851463d, 49433)); + } + + // todo max field should has long as type instead of integer type. + @Ignore + @Test + public void logWithAddLiteralOnGroupKeyAndMaxSubtractLiteralShouldPass() { + JSONObject response = executeJdbcRequest(String.format( + "SELECT gender, Log(age+10) as logAge, max(balance) - 100 as max " + + "FROM %s " + + "WHERE gender = 'm' and age < 22 " + + "GROUP BY gender, age " + + "ORDER BY age", + Index.ACCOUNT.getName())); + + verifySchema(response, + schema("gender", null, "text"), + schema("logAge", "logAge", "double"), + schema("max", "max", "long")); + verifyDataRows(response, + rows("m", 3.4011973816621555d, 49468), + rows("m", 3.4339872044851463d, 49333)); + } + + /** + * The date is in JDBC format. + */ + @Test + public void groupByDateShouldPass() { + JSONObject response = executeJdbcRequest(String.format( + "SELECT birthdate, count(*) as count " + + "FROM %s " + + "WHERE age < 30 " + + "GROUP BY birthdate ", + Index.BANK.getName())); + + verifySchema(response, + schema("birthdate", null, "date"), + schema("count", "count", "integer")); + verifyDataRows(response, + rows("2018-06-23 00:00:00.000", 1)); + } + + @Test + public void groupByDateWithAliasShouldPass() { + JSONObject response = executeJdbcRequest(String.format( + "SELECT birthdate as birth, count(*) as count " + + "FROM %s " + + "WHERE age < 30 " + + "GROUP BY birthdate ", + Index.BANK.getName())); + + verifySchema(response, + schema("birth", "birth", "date"), + schema("count", "count", "integer")); + verifyDataRows(response, + rows("2018-06-23 00:00:00.000", 1)); + } + + private JSONObject executeJdbcRequest(String query) { + return new JSONObject(executeQuery(query, "jdbc")); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/AggregationIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/AggregationIT.java new file mode 100644 index 0000000000..99799addaa --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/AggregationIT.java @@ -0,0 +1,1345 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_DOG; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_GAME_OF_THRONES; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_TYPE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ONLINE; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.rows; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.schema; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyDataRows; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyDataRowsInOrder; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifySchema; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.isEmptyString; +import static org.hamcrest.Matchers.not; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + +public class AggregationIT extends SQLIntegTestCase { + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + loadIndex(Index.GAME_OF_THRONES); + loadIndex(Index.DOG); + loadIndex(Index.ONLINE); + loadIndex(Index.NESTED); + } + + @Test + public void countTest() throws IOException { + + JSONObject result = executeQuery(String.format("SELECT COUNT(*) FROM %s", TEST_INDEX_ACCOUNT)); + Assert.assertThat(getTotalHits(result), equalTo(1000)); + Assert.assertThat(getIntAggregationValue(result, "COUNT(*)", "value"), equalTo(1000)); + } + + @Test + public void countDistinctTest() { + JSONObject response = executeJdbcRequest( + String.format("SELECT COUNT(distinct gender) FROM %s", TEST_INDEX_ACCOUNT)); + + verifySchema(response, schema("COUNT(DISTINCT gender)", null, "integer")); + verifyDataRows(response, rows(2)); + } + + @Test + public void countWithDocsHintTest() throws Exception { + + JSONObject result = + executeQuery(String.format("SELECT /*! DOCS_WITH_AGGREGATION(10) */ count(*) from %s", + TEST_INDEX_ACCOUNT)); + JSONArray hits = (JSONArray) result.query("/hits/hits"); + Assert.assertThat(hits.length(), equalTo(10)); + } + + @Test + public void sumTest() throws IOException { + + JSONObject result = + executeQuery(String.format("SELECT SUM(balance) FROM %s", TEST_INDEX_ACCOUNT)); + Assert.assertThat(getTotalHits(result), equalTo(1000)); + Assert.assertThat(getDoubleAggregationValue(result, "SUM(balance)", "value"), + equalTo(25714837.0)); + } + + @Test + public void minTest() throws IOException { + + JSONObject result = executeQuery(String.format("SELECT MIN(age) FROM %s", TEST_INDEX_ACCOUNT)); + Assert.assertThat(getTotalHits(result), equalTo(1000)); + Assert.assertThat(getDoubleAggregationValue(result, "MIN(age)", "value"), equalTo(20.0)); + } + + @Test + public void maxTest() throws IOException { + + JSONObject result = executeQuery(String.format("SELECT MAX(age) FROM %s", TEST_INDEX_ACCOUNT)); + Assert.assertThat(getTotalHits(result), equalTo(1000)); + Assert.assertThat(getDoubleAggregationValue(result, "MAX(age)", "value"), equalTo(40.0)); + } + + @Test + public void avgTest() throws IOException { + + JSONObject result = executeQuery(String.format("SELECT AVG(age) FROM %s", TEST_INDEX_ACCOUNT)); + Assert.assertThat(getTotalHits(result), equalTo(1000)); + Assert.assertThat(getDoubleAggregationValue(result, "AVG(age)", "value"), equalTo(30.171)); + } + + @Test + public void statsTest() throws IOException { + + JSONObject result = + executeQuery(String.format("SELECT STATS(age) FROM %s", TEST_INDEX_ACCOUNT)); + Assert.assertThat(getTotalHits(result), equalTo(1000)); + Assert.assertThat(getIntAggregationValue(result, "STATS(age)", "count"), equalTo(1000)); + Assert.assertThat(getDoubleAggregationValue(result, "STATS(age)", "min"), equalTo(20.0)); + Assert.assertThat(getDoubleAggregationValue(result, "STATS(age)", "max"), equalTo(40.0)); + Assert.assertThat(getDoubleAggregationValue(result, "STATS(age)", "avg"), equalTo(30.171)); + Assert.assertThat(getDoubleAggregationValue(result, "STATS(age)", "sum"), equalTo(30171.0)); + } + + @Test + public void extendedStatsTest() throws IOException { + + JSONObject result = executeQuery(String.format("SELECT EXTENDED_STATS(age) FROM %s", + TEST_INDEX_ACCOUNT)); + Assert.assertThat(getTotalHits(result), equalTo(1000)); + Assert + .assertThat(getDoubleAggregationValue(result, "EXTENDED_STATS(age)", "min"), equalTo(20.0)); + Assert + .assertThat(getDoubleAggregationValue(result, "EXTENDED_STATS(age)", "max"), equalTo(40.0)); + Assert.assertThat(getDoubleAggregationValue(result, "EXTENDED_STATS(age)", "avg"), + equalTo(30.171)); + Assert.assertThat(getDoubleAggregationValue(result, "EXTENDED_STATS(age)", "sum"), + equalTo(30171.0)); + Assert.assertThat(getDoubleAggregationValue(result, "EXTENDED_STATS(age)", "sum_of_squares"), + equalTo(946393.0)); + Assert.assertEquals(6.008640362012022, + getDoubleAggregationValue(result, "EXTENDED_STATS(age)", "std_deviation"), 0.0001); + Assert.assertEquals(36.10375899999996, + getDoubleAggregationValue(result, "EXTENDED_STATS(age)", "variance"), 0.0001); + } + + @Test + public void percentileTest() throws IOException { + + JSONObject result = + executeQuery(String.format("SELECT PERCENTILES(age) FROM %s", TEST_INDEX_ACCOUNT)); + Assert.assertThat(getTotalHits(result), equalTo(1000)); + Assert + .assertEquals(20.0, getDoubleAggregationValue(result, "PERCENTILES(age)", "values", "1.0"), + 0.001); + Assert + .assertEquals(21.0, getDoubleAggregationValue(result, "PERCENTILES(age)", "values", "5.0"), + 0.001); + Assert + .assertEquals(25.0, getDoubleAggregationValue(result, "PERCENTILES(age)", "values", "25.0"), + 0.001); + // All percentiles are approximations calculated by t-digest, however, P50 has the widest distribution (not sure why) + Assert + .assertEquals(30.5, getDoubleAggregationValue(result, "PERCENTILES(age)", "values", "50.0"), + 0.6); + Assert + .assertEquals(35.0, getDoubleAggregationValue(result, "PERCENTILES(age)", "values", "75.0"), + 0.001); + Assert + .assertEquals(39.0, getDoubleAggregationValue(result, "PERCENTILES(age)", "values", "95.0"), + 0.001); + Assert + .assertEquals(40.0, getDoubleAggregationValue(result, "PERCENTILES(age)", "values", "99.0"), + 0.001); + } + + @Test + public void percentileTestSpecific() throws IOException { + + JSONObject result = executeQuery(String.format("SELECT PERCENTILES(age,25.0,75.0) FROM %s", + TEST_INDEX_ACCOUNT)); + + Assert.assertThat(getTotalHits(result), equalTo(1000)); + Assert.assertEquals(25.0, + getDoubleAggregationValue(result, "PERCENTILES(age,25.0,75.0)", "values", "25.0"), 0.001); + Assert.assertEquals(35.0, + getDoubleAggregationValue(result, "PERCENTILES(age,25.0,75.0)", "values", "75.0"), 0.001); + } + + @Test + public void aliasTest() throws IOException { + + JSONObject result = executeQuery(String.format("SELECT COUNT(*) AS mycount FROM %s", + TEST_INDEX_ACCOUNT)); + Assert.assertThat(getTotalHits(result), equalTo(1000)); + Assert.assertThat(getIntAggregationValue(result, "mycount", "value"), equalTo(1000)); + } + + @Test + public void groupByTest() throws Exception { + JSONObject result = executeQuery(String.format("SELECT COUNT(*) FROM %s GROUP BY gender", + TEST_INDEX_ACCOUNT)); + assertResultForGroupByTest(result); + } + + @Test + public void groupByUsingTableAliasTest() throws Exception { + JSONObject result = executeQuery(String.format("SELECT COUNT(*) FROM %s a GROUP BY a.gender", + TEST_INDEX_ACCOUNT)); + assertResultForGroupByTest(result); + } + + @Test + public void groupByUsingTableNamePrefixTest() throws Exception { + JSONObject result = executeQuery(String.format( + "SELECT COUNT(*) FROM %s GROUP BY elasticsearch-sql_test_index_account.gender", + TEST_INDEX_ACCOUNT + )); + assertResultForGroupByTest(result); + } + + private void assertResultForGroupByTest(JSONObject result) { + Assert.assertThat(getTotalHits(result), equalTo(1000)); + JSONObject gender = getAggregation(result, "gender"); + Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); + + final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); + final int maleBucketId = isMaleFirst ? 0 : 1; + final int femaleBucketId = isMaleFirst ? 1 : 0; + + final String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); + final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); + + Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); + Assert.assertThat(gender.query(maleBucketPrefix + "/COUNT(*)/value"), equalTo(507)); + Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); + Assert.assertThat(gender.query(femaleBucketPrefix + "/COUNT(*)/value"), equalTo(493)); + } + + @Test + public void groupByHavingTest() throws Exception { + JSONObject result = executeQuery(String.format( + "SELECT gender " + + "FROM %s " + + "GROUP BY gender " + + "HAVING COUNT(*) > 0", TEST_INDEX_ACCOUNT)); + assertResultForGroupByHavingTest(result); + } + + @Test + public void groupByHavingUsingTableAliasTest() throws Exception { + JSONObject result = executeQuery(String.format( + "SELECT a.gender " + + "FROM %s a " + + "GROUP BY a.gender " + + "HAVING COUNT(*) > 0", TEST_INDEX_ACCOUNT)); + assertResultForGroupByHavingTest(result); + } + + @Test + public void groupByHavingUsingTableNamePrefixTest() throws Exception { + JSONObject result = executeQuery(String.format( + "SELECT elasticsearch-sql_test_index_account.gender " + + "FROM %s " + + "GROUP BY elasticsearch-sql_test_index_account.gender " + + "HAVING COUNT(*) > 0", TEST_INDEX_ACCOUNT)); + assertResultForGroupByHavingTest(result); + } + + private void assertResultForGroupByHavingTest(JSONObject result) { + Assert.assertThat(getTotalHits(result), equalTo(1000)); + JSONObject gender = getAggregation(result, "gender"); + Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); + + final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); + final int maleBucketId = isMaleFirst ? 0 : 1; + final int femaleBucketId = isMaleFirst ? 1 : 0; + + final String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); + final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); + + Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); + Assert.assertThat(gender.query(maleBucketPrefix + "/count_0/value"), equalTo(507)); + Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); + Assert.assertThat(gender.query(femaleBucketPrefix + "/count_0/value"), equalTo(493)); + } + + @Ignore //todo VerificationException: table alias or field name missing + @Test + public void groupBySubqueryTest() throws Exception { + + JSONObject result = executeQuery(String.format( + "SELECT COUNT(*) FROM %s " + + "WHERE firstname IN (SELECT firstname FROM %s) " + + "GROUP BY gender", + TEST_INDEX_ACCOUNT, TEST_INDEX_ACCOUNT)); + Assert.assertThat(getTotalHits(result), equalTo(1000)); + JSONObject gender = getAggregation(result, "gender"); + Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); + + final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); + final int maleBucketId = isMaleFirst ? 0 : 1; + final int femaleBucketId = isMaleFirst ? 1 : 0; + + final String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); + final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); + + Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); + Assert.assertThat(gender.query(maleBucketPrefix + "/COUNT(*)/value"), equalTo(507)); + Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); + Assert.assertThat(gender.query(femaleBucketPrefix + "/COUNT(*)/value"), equalTo(493)); + } + + @Test + public void postFilterTest() throws Exception { + + JSONObject result = executeQuery(String.format("SELECT /*! POST_FILTER({\\\"term\\\":" + + "{\\\"gender\\\":\\\"m\\\"}}) */ COUNT(*) FROM %s GROUP BY gender", + TEST_INDEX_ACCOUNT)); + Assert.assertThat(getTotalHits(result), equalTo(507)); + JSONObject gender = getAggregation(result, "gender"); + Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); + + final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); + final int maleBucketId = isMaleFirst ? 0 : 1; + final int femaleBucketId = isMaleFirst ? 1 : 0; + + final String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); + final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); + + Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); + Assert.assertThat(gender.query(maleBucketPrefix + "/COUNT(*)/value"), equalTo(507)); + Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); + Assert.assertThat(gender.query(femaleBucketPrefix + "/COUNT(*)/value"), equalTo(493)); + } + + @Test + public void multipleGroupByTest() throws Exception { + + JSONObject result = executeQuery(String.format("SELECT COUNT(*) FROM %s GROUP BY gender," + + " terms('field'='age','size'=200,'alias'='age')", + TEST_INDEX_ACCOUNT)); + Assert.assertThat(getTotalHits(result), equalTo(1000)); + JSONObject gender = getAggregation(result, "gender"); + Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); + + final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); + final int maleBucketId = isMaleFirst ? 0 : 1; + final int femaleBucketId = isMaleFirst ? 1 : 0; + + final String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); + final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); + + final JSONArray mAgeBuckets = (JSONArray) (gender.optQuery(maleBucketPrefix + "/age/buckets")); + final JSONArray fAgeBuckets = + (JSONArray) (gender.optQuery(femaleBucketPrefix + "/age/buckets")); + + final Set expectedAges = + IntStream.range(20, 41).boxed().collect(Collectors.toCollection(HashSet::new)); + Assert.assertThat(mAgeBuckets.length(), equalTo(expectedAges.size())); + Assert.assertThat(fAgeBuckets.length(), equalTo(expectedAges.size())); + + final Set actualAgesM = new HashSet<>(expectedAges.size()); + final Set actualAgesF = new HashSet<>(expectedAges.size()); + mAgeBuckets.iterator() + .forEachRemaining(json -> actualAgesM.add(((JSONObject) json).getInt("key"))); + fAgeBuckets.iterator() + .forEachRemaining(json -> actualAgesF.add(((JSONObject) json).getInt("key"))); + + Assert.assertThat(actualAgesM, equalTo(expectedAges)); + Assert.assertThat(actualAgesF, equalTo(expectedAges)); + } + + @Test + public void multipleGroupBysWithSize() throws Exception { + + JSONObject result = executeQuery(String.format("SELECT COUNT(*) FROM %s GROUP BY gender," + + " terms('alias'='ageAgg','field'='age','size'=3)", + TEST_INDEX_ACCOUNT)); + Assert.assertThat(getTotalHits(result), equalTo(1000)); + JSONObject gender = getAggregation(result, "gender"); + Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); + + final JSONArray mAgeBuckets = (JSONArray) (gender.optQuery("/buckets/0/ageAgg/buckets")); + final JSONArray fAgeBuckets = (JSONArray) (gender.optQuery("/buckets/0/ageAgg/buckets")); + + Assert.assertThat(mAgeBuckets.length(), equalTo(3)); + Assert.assertThat(fAgeBuckets.length(), equalTo(3)); + } + + @Test + public void termsWithSize() throws Exception { + + JSONObject result = executeQuery(String.format("SELECT COUNT(*) FROM %s GROUP BY terms" + + "('alias'='ageAgg','field'='age','size'=3)", + TEST_INDEX_ACCOUNT)); + Assert.assertThat(getTotalHits(result), equalTo(1000)); + JSONObject gender = getAggregation(result, "ageAgg"); + Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(3)); + } + + @Test + public void termsWithMissing() throws Exception { + + JSONObject result = executeQuery(String.format("SELECT count(*) FROM %s GROUP BY terms" + + "('alias'='nick','field'='nickname','missing'='no_nickname')", + TEST_INDEX_GAME_OF_THRONES)); + JSONObject nick = getAggregation(result, "nick"); + + Optional noNicknameBucket = Optional.empty(); + Iterator iter = nick.getJSONArray("buckets").iterator(); + while (iter.hasNext()) { + JSONObject bucket = (JSONObject) iter.next(); + if (bucket.getString("key").equals("no_nickname")) { + noNicknameBucket = Optional.of(bucket); + Assert.assertThat(bucket.getInt("doc_count"), equalTo(6)); + } + } + Assert.assertTrue(noNicknameBucket.isPresent()); + } + + @Test + public void termsWithOrder() throws Exception { + + final String dog1 = "snoopy"; + final String dog2 = "rex"; + + JSONObject result = executeQuery(String.format("SELECT count(*) FROM %s GROUP BY terms" + + "('field'='dog_name', 'alias'='dog_name', 'order'='desc')", + TEST_INDEX_DOG)); + JSONObject dogName = getAggregation(result, "dog_name"); + + String firstDog = (String) (dogName.optQuery("/buckets/0/key")); + String secondDog = (String) (dogName.optQuery("/buckets/1/key")); + Assert.assertThat(firstDog, equalTo(dog1)); + Assert.assertThat(secondDog, equalTo(dog2)); + + result = executeQuery(String.format("SELECT count(*) FROM %s GROUP BY terms" + + "('field'='dog_name', 'alias'='dog_name', 'order'='asc')", TEST_INDEX_DOG)); + + dogName = getAggregation(result, "dog_name"); + + firstDog = (String) (dogName.optQuery("/buckets/0/key")); + secondDog = (String) (dogName.optQuery("/buckets/1/key")); + Assert.assertThat(firstDog, equalTo(dog2)); + Assert.assertThat(secondDog, equalTo(dog1)); + } + + @Test + public void orderByAscTest() { + JSONObject response = executeJdbcRequest(String.format("SELECT COUNT(*) FROM %s " + + "GROUP BY gender ORDER BY COUNT(*)", TEST_INDEX_ACCOUNT)); + + verifySchema(response, schema("COUNT(*)", null, "integer")); + verifyDataRows(response, + rows(493), + rows(507)); + } + + @Test + public void orderByAliasAscTest() { + JSONObject response = executeJdbcRequest(String.format("SELECT COUNT(*) as count FROM %s " + + "GROUP BY gender ORDER BY count", TEST_INDEX_ACCOUNT)); + + verifySchema(response, schema("count", "count", "integer")); + verifyDataRowsInOrder(response, + rows(493), + rows(507)); + } + + @Test + public void orderByDescTest() throws IOException { + JSONObject response = executeJdbcRequest(String.format("SELECT COUNT(*) FROM %s " + + "GROUP BY gender ORDER BY COUNT(*) DESC", TEST_INDEX_ACCOUNT)); + + verifySchema(response, schema("COUNT(*)", null, "integer")); + verifyDataRowsInOrder(response, + rows(507), + rows(493)); + } + + @Test + public void orderByAliasDescTest() throws IOException { + JSONObject response = executeJdbcRequest(String.format("SELECT COUNT(*) as count FROM %s " + + "GROUP BY gender ORDER BY count DESC", TEST_INDEX_ACCOUNT)); + + verifySchema(response, schema("count", "count", "integer")); + verifyDataRowsInOrder(response, + rows(507), + rows(493)); + } + + @Test + public void limitTest() throws IOException { + JSONObject response = executeJdbcRequest(String.format("SELECT COUNT(*) FROM %s " + + "GROUP BY age ORDER BY COUNT(*) LIMIT 5", TEST_INDEX_ACCOUNT)); + + verifySchema(response, schema("COUNT(*)", null, "integer")); + verifyDataRowsInOrder(response, + rows(35), + rows(39), + rows(39), + rows(42), + rows(42)); + } + + @Test + public void countGroupByRange() throws IOException { + + JSONObject result = executeQuery(String.format("SELECT COUNT(age) FROM %s" + + " GROUP BY range(age, 20,25,30,35,40)", TEST_INDEX_ACCOUNT)); + JSONObject ageAgg = getAggregation(result, "range(age,20,25,30,35,40)"); + JSONArray buckets = ageAgg.getJSONArray("buckets"); + Assert.assertThat(buckets.length(), equalTo(4)); + + final int[] expectedResults = new int[] {225, 226, 259, 245}; + + for (int i = 0; i < expectedResults.length; ++i) { + + Assert.assertThat(buckets.query(String.format(Locale.ROOT, "/%d/COUNT(age)/value", i)), + equalTo(expectedResults[i])); + } + } + + /** + * http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-aggregations-bucket-datehistogram-aggregation.html + */ + @Test + public void countGroupByDateTest() throws IOException { + + String result = + explainQuery(String.format("select insert_time from %s group by date_histogram" + + "('field'='insert_time','fixed_interval'='1h','format'='yyyy-MM','min_doc_count'=5) ", + TEST_INDEX_ONLINE)); + Assert.assertThat(result.replaceAll("\\s+", ""), + containsString("{\"date_histogram\":{\"field\":\"insert_time\",\"format\":\"yyyy-MM\"," + + "\"fixed_interval\":\"1h\",\"offset\":0,\"order\":{\"_key\":\"asc\"},\"keyed\":false," + + "\"min_doc_count\":5}")); + } + + @Test + public void countGroupByDateTestWithAlias() throws IOException { + String result = + explainQuery(String.format("select insert_time from %s group by date_histogram" + + "('field'='insert_time','fixed_interval'='1h','format'='yyyy-MM','alias'='myAlias')", + TEST_INDEX_ONLINE)); + Assert.assertThat(result.replaceAll("\\s+", ""), + containsString("myAlias\":{\"date_histogram\":{\"field\":\"insert_time\"," + + "\"format\":\"yyyy-MM\",\"fixed_interval\":\"1h\"")); + } + +// /** +// * http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-aggregations-bucket-daterange-aggregation.html +// */ +// @Test +// public void countDateRangeTest() throws IOException, SqlParseException, SQLFeatureNotSupportedException { +// String result = explainQuery(String.format("select online from %s group by date_range(field='insert_time'," + +// "'format'='yyyy-MM-dd' ,'2014-08-18','2014-08-17','now-8d','now-7d','now-6d','now')", +// TEST_INDEX_ONLINE)); +// // TODO: fix the query or fix the code for the query to work +// } + + @Test + public void topHitTest() throws IOException { + + String query = String + .format("select topHits('size'=3,age='desc') from %s group by gender", TEST_INDEX_ACCOUNT); + JSONObject result = executeQuery(query); + JSONObject gender = getAggregation(result, "gender"); + Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); + + final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); + final int maleBucketId = isMaleFirst ? 0 : 1; + final int femaleBucketId = isMaleFirst ? 1 : 0; + + final String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); + final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); + + Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); + Assert.assertThat(gender.query(maleBucketPrefix + "/topHits(size=3,age=desc)/hits/total/value"), + equalTo(507)); + Assert.assertThat( + gender.query(maleBucketPrefix + "/topHits(size=3,age=desc)/hits/total/relation"), + equalTo("eq")); + Assert.assertThat( + ((JSONArray) gender.query(maleBucketPrefix + "/topHits(size=3,age=desc)/hits/hits")) + .length(), + equalTo(3)); + Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); + Assert + .assertThat(gender.query(femaleBucketPrefix + "/topHits(size=3,age=desc)/hits/total/value"), + equalTo(493)); + Assert.assertThat( + gender.query(femaleBucketPrefix + "/topHits(size=3,age=desc)/hits/total/relation"), + equalTo("eq")); + Assert.assertThat( + ((JSONArray) gender.query(femaleBucketPrefix + "/topHits(size=3,age=desc)/hits/hits")) + .length(), + equalTo(3)); + } + + @Test + public void topHitTest_WithInclude() throws IOException { + + String query = + String.format("select topHits('size'=3,age='desc','include'=age) from %s group by gender", + TEST_INDEX_ACCOUNT); + JSONObject result = executeQuery(query); + JSONObject gender = getAggregation(result, "gender"); + Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); + + final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); + final int maleBucketId = isMaleFirst ? 0 : 1; + final int femaleBucketId = isMaleFirst ? 1 : 0; + + final String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); + final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); + + Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); + Assert.assertThat( + gender.query(maleBucketPrefix + "/topHits(size=3,age=desc,include=age)/hits/total/value"), + equalTo(507)); + Assert.assertThat(gender + .query(maleBucketPrefix + "/topHits(size=3,age=desc,include=age)/hits/total/relation"), + equalTo("eq")); + Assert.assertThat(((JSONArray) gender.query( + maleBucketPrefix + "/topHits(size=3,age=desc,include=age)/hits/hits")).length(), + equalTo(3)); + + Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); + Assert.assertThat( + gender.query(femaleBucketPrefix + "/topHits(size=3,age=desc,include=age)/hits/total/value"), + equalTo(493)); + Assert.assertThat(gender + .query(femaleBucketPrefix + "/topHits(size=3,age=desc,include=age)/hits/total/relation"), + equalTo("eq")); + Assert.assertThat(((JSONArray) gender.query( + femaleBucketPrefix + "/topHits(size=3,age=desc,include=age)/hits/hits")).length(), + equalTo(3)); + + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < 3; ++j) { + JSONObject source = (JSONObject) gender.query(String.format(Locale.ROOT, + "/buckets/%d/topHits(size=3,age=desc,include=age)/hits/hits/%d/_source", i, j)); + Assert.assertThat(source.length(), equalTo(1)); + Assert.assertTrue(source.has("age")); + Assert.assertThat(source.getInt("age"), equalTo(40)); + } + } + } + + @Test + public void topHitTest_WithIncludeTwoFields() throws IOException { + + String query = + String.format("select topHits('size'=3,'include'='age,firstname',age='desc') from %s " + + "group by gender", TEST_INDEX_ACCOUNT); + JSONObject result = executeQuery(query); + JSONObject gender = getAggregation(result, "gender"); + Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); + + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < 3; ++j) { + JSONObject source = (JSONObject) gender.query(String.format(Locale.ROOT, + "/buckets/%d/topHits(size=3,include=age,firstname,age=desc)/hits/hits/%d/_source", i, + j)); + Assert.assertThat(source.length(), equalTo(2)); + Assert.assertTrue(source.has("age")); + Assert.assertThat(source.getInt("age"), equalTo(40)); + Assert.assertTrue(source.has("firstname")); + final String name = source.getString("firstname"); + Assert.assertThat(name, not(isEmptyString())); + } + } + } + + @Test + public void topHitTest_WithExclude() throws IOException { + + String query = String.format("select topHits('size'=3,'exclude'='lastname',age='desc') from " + + "%s group by gender", TEST_INDEX_ACCOUNT); + JSONObject result = executeQuery(query); + JSONObject gender = getAggregation(result, "gender"); + Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); + + final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); + final int maleBucketId = isMaleFirst ? 0 : 1; + final int femaleBucketId = isMaleFirst ? 1 : 0; + + final String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); + final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); + + Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); + Assert.assertThat(gender + .query(maleBucketPrefix + "/topHits(size=3,exclude=lastname,age=desc)/hits/total/value"), + equalTo(507)); + Assert.assertThat(gender + .query(maleBucketPrefix + "/topHits(size=3,exclude=lastname,age=desc)/hits/total/relation"), + equalTo("eq")); + Assert.assertThat(((JSONArray) gender.query( + maleBucketPrefix + "/topHits(size=3,exclude=lastname,age=desc)/hits/hits")).length(), + equalTo(3)); + + Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); + Assert.assertThat(gender + .query(femaleBucketPrefix + "/topHits(size=3,exclude=lastname,age=desc)/hits/total/value"), + equalTo(493)); + Assert.assertThat(gender.query( + femaleBucketPrefix + "/topHits(size=3,exclude=lastname,age=desc)/hits/total/relation"), + equalTo("eq")); + Assert.assertThat(((JSONArray) gender.query( + femaleBucketPrefix + "/topHits(size=3,exclude=lastname,age=desc)/hits/hits")).length(), + equalTo(3)); + + final Set expectedFields = new HashSet<>(Arrays.asList( + "account_number", + "firstname", + "address", + "balance", + "gender", + "city", + "employer", + "state", + "age", + "email" + )); + + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < 3; ++j) { + JSONObject source = (JSONObject) gender.query(String.format(Locale.ROOT, + "/buckets/%d/topHits(size=3,exclude=lastname,age=desc)/hits/hits/%d/_source", i, j)); + Assert.assertThat(source.length(), equalTo(expectedFields.size())); + Assert.assertFalse(source.has("lastname")); + Assert.assertThat(source.keySet().containsAll(expectedFields), equalTo(true)); + } + } + } + + //region not migrated + + // script on metric aggregation tests. uncomment if your elastic has scripts enable (disabled by default) +// @Test +// public void sumWithScriptTest() throws IOException, SqlParseException, SQLFeatureNotSupportedException { +// Aggregations result = query(String.format("SELECT SUM(script('','doc[\\'balance\\'].value + doc[\\'balance\\'].value')) as doubleSum FROM %s", TEST_INDEX)); +// Sum sum = result.get("doubleSum"); +// assertThat(sum.getValue(), equalTo(25714837.0*2)); +// } +// +// @Test +// public void sumWithImplicitScriptTest() throws IOException, SqlParseException, SQLFeatureNotSupportedException { +// Aggregations result = query(String.format("SELECT SUM(balance + balance) as doubleSum FROM %s", TEST_INDEX)); +// Sum sum = result.get("doubleSum"); +// assertThat(sum.getValue(), equalTo(25714837.0*2)); +// } +// +// @Test +// public void sumWithScriptTestNoAlias() throws IOException, SqlParseException, SQLFeatureNotSupportedException { +// Aggregations result = query(String.format("SELECT SUM(balance + balance) FROM %s", TEST_INDEX)); +// Sum sum = result.get("SUM(script=script(balance + balance,doc('balance').value + doc('balance').value))"); +// assertThat(sum.getValue(), equalTo(25714837.0*2)); +// } +// +// @Test +// public void scriptedMetricAggregation() throws SQLFeatureNotSupportedException, SqlParseException { +// Aggregations result = query ("select scripted_metric('map_script'='if(doc[\\'balance\\'].value > 49670){ if(!_agg.containsKey(\\'ages\\')) { _agg.put(\\'ages\\',doc[\\'age\\'].value); } " + +// "else { _agg.put(\\'ages\\',_agg.get(\\'ages\\')+doc[\\'age\\'].value); }}'," + +// "'reduce_script'='sumThem = 0; for (a in _aggs) { if(a.containsKey(\\'ages\\')){ sumThem += a.get(\\'ages\\');} }; return sumThem;') as wierdSum from " + TEST_INDEX + ""); +// ScriptedMetric metric = result.get("wierdSum"); +// Assert.assertEquals(136L,metric.aggregation()); +// } +// +// @Test +// public void scriptedMetricConcatWithStringParamAndReduceParamAggregation() throws SQLFeatureNotSupportedException, SqlParseException { +// String query = "select scripted_metric(\n" + +// " 'init_script' = '_agg[\"concat\"]=[] ',\n" + +// " 'map_script'='_agg.concat.add(doc[field].value)' ,\n" + +// " 'combine_script'='return _agg.concat.join(delim);',\t\t\t\t\n" + +// " 'reduce_script'='_aggs.removeAll(\"\"); return _aggs.join(delim)'," + +// "'@field' = 'name.firstname' , '@delim'=';',@reduce_delim =';' ) as all_characters \n" + +// "from "+TEST_INDEX+""; +// Aggregations result = query (query); +// ScriptedMetric metric = result.get("all_characters"); +// List names = Arrays.asList(metric.aggregation().toString().split(";")); +// +// +// Assert.assertEquals(4,names.size()); +// String[] expectedNames = new String[]{"brandon","daenerys","eddard","jaime"}; +// for(String name : expectedNames){ +// Assert.assertTrue("not contains:" + name,names.contains(name)); +// } +// } +// +// @Test +// public void scriptedMetricAggregationWithNumberParams() throws SQLFeatureNotSupportedException, SqlParseException { +// Aggregations result = query ("select scripted_metric('map_script'='if(doc[\\'balance\\'].value > 49670){ if(!_agg.containsKey(\\'ages\\')) { _agg.put(\\'ages\\',doc[\\'age\\'].value+x); } " + +// "else { _agg.put(\\'ages\\',_agg.get(\\'ages\\')+doc[\\'age\\'].value+x); }}'," + +// "'reduce_script'='sumThem = 0; for (a in _aggs) { if(a.containsKey(\\'ages\\')){ sumThem += a.get(\\'ages\\');} }; return sumThem;'" + +// ",'@x'=3) as wierdSum from " + TEST_INDEX + ""); +// ScriptedMetric metric = result.get("wierdSum"); +// Assert.assertEquals(148L,metric.aggregation()); +// } +// + +// @Test +// public void topHitTest_WithIncludeAndExclude() throws IOException, SqlParseException, SQLFeatureNotSupportedException { +// Aggregations result = query(String.format("select topHits('size'=3,'exclude'='lastname','include'='firstname,lastname',age='desc') from %s group by gender ", TEST_INDEX_ACCOUNT)); +// List buckets = ((Terms) (result.asList().get(0))).getBuckets(); +// for (Terms.Bucket bucket : buckets) { +// SearchHits hits = ((InternalTopHits) bucket.getAggregations().asList().get(0)).getHits(); +// for (SearchHit hit : hits) { +// Set fields = hit.getSourceAsMap().keySet(); +// Assert.assertEquals(1, fields.size()); +// Assert.assertTrue(fields.contains("firstname")); +// } +// } +// } +// +// private Aggregations query(String query) throws SqlParseException, SQLFeatureNotSupportedException { +// SqlElasticSearchRequestBuilder select = getSearchRequestBuilder(query); +// return ((SearchResponse)select.get()).getAggregations(); +// } +// +// private SqlElasticSearchRequestBuilder getSearchRequestBuilder(String query) throws SqlParseException, SQLFeatureNotSupportedException { +// SearchDao searchDao = MainTestSuite.getSearchDao(); +// return (SqlElasticSearchRequestBuilder) searchDao.explain(query).explain(); +// } +// +// @Test +// public void testFromSizeWithAggregations() throws Exception { +// final String query1 = String.format("SELECT /*! DOCS_WITH_AGGREGATION(0,1) */" + +// " account_number FROM %s GROUP BY gender", TEST_INDEX_ACCOUNT); +// SearchResponse response1 = (SearchResponse) getSearchRequestBuilder(query1).get(); +// +// Assert.assertEquals(1, response1.getHits().getHits().length); +// Terms gender1 = response1.getAggregations().get("gender"); +// Assert.assertEquals(2, gender1.getBuckets().size()); +// Object account1 = response1.getHits().getHits()[0].getSourceAsMap().get("account_number"); +// +// final String query2 = String.format("SELECT /*! DOCS_WITH_AGGREGATION(1,1) */" + +// " account_number FROM %s GROUP BY gender", TEST_INDEX_ACCOUNT); +// SearchResponse response2 = (SearchResponse) getSearchRequestBuilder(query2).get(); +// +// Assert.assertEquals(1, response2.getHits().getHits().length); +// Terms gender2 = response2.getAggregations().get("gender"); +// Assert.assertEquals(2, gender2.getBuckets().size()); +// Object account2 = response2.getHits().getHits()[0].getSourceAsMap().get("account_number"); +// +// Assert.assertEquals(response1.getHits().getTotalHits(), response2.getHits().getTotalHits()); +// Assert.assertNotEquals(account1, account2); +// } +// +// @Test +// public void testSubAggregations() throws Exception { +// Set expectedAges = new HashSet<>(ContiguousSet.create(Range.closed(20, 40), DiscreteDomain.integers())); +// final String query = String.format("SELECT /*! DOCS_WITH_AGGREGATION(10) */" + +// " * FROM %s GROUP BY (gender, terms('field'='age','size'=200,'alias'='age')), (state) LIMIT 200,200", TEST_INDEX_ACCOUNT); +// +// Map> buckets = new HashMap<>(); +// +// SqlElasticSearchRequestBuilder select = getSearchRequestBuilder(query); +// SearchResponse response = (SearchResponse) select.get(); +// Aggregations result = response.getAggregations(); +// +// Terms gender = result.get("gender"); +// for(Terms.Bucket genderBucket : gender.getBuckets()) { +// String genderKey = genderBucket.getKey().toString(); +// buckets.put(genderKey, new HashSet()); +// Terms ageBuckets = (Terms) genderBucket.getAggregations().get("age"); +// for(Terms.Bucket ageBucket : ageBuckets.getBuckets()) { +// buckets.get(genderKey).add(Integer.parseInt(ageBucket.getKey().toString())); +// } +// } +// +// Assert.assertEquals(2, buckets.keySet().size()); +// Assert.assertEquals(expectedAges, buckets.get("m")); +// Assert.assertEquals(expectedAges, buckets.get("f")); +// +// Terms state = result.get("state.keyword"); +// for(Terms.Bucket stateBucket : state.getBuckets()) { +// if(stateBucket.getKey().toString().equalsIgnoreCase("ak")) { +// Assert.assertTrue("There are 22 entries for state ak", stateBucket.getDocCount() == 22); +// } +// } +// +// Assert.assertEquals(response.getHits().getTotalHits(), 1000); +// Assert.assertEquals(response.getHits().getHits().length, 10); +// } +// +// @Test +// public void testSimpleSubAggregations() throws Exception { +// final String query = String.format("SELECT /*! DOCS_WITH_AGGREGATION(10) */ * FROM %s GROUP BY (gender), (state) ", TEST_INDEX_ACCOUNT); +// +// SqlElasticSearchRequestBuilder select = getSearchRequestBuilder(query); +// SearchResponse response = (SearchResponse) select.get(); +// Aggregations result = response.getAggregations(); +// +// Terms gender = result.get("gender"); +// for(Terms.Bucket genderBucket : gender.getBuckets()) { +// String genderKey = genderBucket.getKey().toString(); +// Assert.assertTrue("Gender should be m or f", genderKey.equals("m") || genderKey.equals("f")); +// } +// +// Assert.assertEquals(2, gender.getBuckets().size()); +// +// Terms state = result.get("state.keyword"); +// for(Terms.Bucket stateBucket : state.getBuckets()) { +// if(stateBucket.getKey().toString().equalsIgnoreCase("ak")) { +// Assert.assertTrue("There are 22 entries for state ak", stateBucket.getDocCount() == 22); +// } +// } +// +// Assert.assertEquals(response.getHits().getTotalHits(), 1000); +// Assert.assertEquals(response.getHits().getHits().length, 10); +// } +// +// @Test +// public void geoHashGrid() throws SQLFeatureNotSupportedException, SqlParseException { +// Aggregations result = query(String.format("SELECT COUNT(*) FROM %s/location GROUP BY geohash_grid(field='center',precision=5) ", TEST_INDEX_LOCATION)); +// InternalGeoHashGrid grid = result.get("geohash_grid(field=center,precision=5)"); +// Collection buckets = grid.getBuckets(); +// for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) { +// Assert.assertTrue(bucket.getKeyAsString().equals("w2fsm") || bucket.getKeyAsString().equals("w0p6y") ); +// Assert.assertEquals(1,bucket.getDocCount()); +// } +// } +// +// @Test +// public void geoBounds() throws SQLFeatureNotSupportedException, SqlParseException { +// Aggregations result = query(String.format("SELECT * FROM %s/location GROUP BY geo_bounds(field='center',alias='bounds') ", TEST_INDEX_LOCATION)); +// InternalGeoBounds bounds = result.get("bounds"); +// Assert.assertEquals(0.5,bounds.bottomRight().getLat(),0.001); +// Assert.assertEquals(105.0,bounds.bottomRight().getLon(),0.001); +// Assert.assertEquals(5.0,bounds.topLeft().getLat(),0.001); +// Assert.assertEquals(100.5,bounds.topLeft().getLon(),0.001); +// } +// +// @Test +// public void groupByOnNestedFieldTest() throws Exception { +// Aggregations result = query(String.format("SELECT COUNT(*) FROM %s GROUP BY nested(message.info)", TEST_INDEX_NESTED_TYPE)); +// InternalNested nested = result.get("message.info@NESTED"); +// Terms infos = nested.getAggregations().get("message.info"); +// Assert.assertEquals(3,infos.getBuckets().size()); +// for(Terms.Bucket bucket : infos.getBuckets()) { +// String key = bucket.getKey().toString(); +// long count = ((ValueCount) bucket.getAggregations().get("COUNT(*)")).getValue(); +// if(key.equalsIgnoreCase("a")) { +// Assert.assertEquals(2, count); +// } +// else if(key.equalsIgnoreCase("c")) { +// Assert.assertEquals(2, count); +// } +// else if(key.equalsIgnoreCase("b")) { +// Assert.assertEquals(1, count); +// } +// else { +// throw new Exception(String.format("Unexpected key. expected: a OR b OR c . found: %s", key)); +// } +// } +// } +// +// @Test +// public void groupByTestWithFilter() throws Exception { +// Aggregations result = query(String.format("SELECT COUNT(*) FROM %s GROUP BY filter(gender='m'),gender", TEST_INDEX_ACCOUNT)); +// InternalFilter filter = result.get("filter(gender = 'm')@FILTER"); +// Terms gender = filter.getAggregations().get("gender"); +// +// for(Terms.Bucket bucket : gender.getBuckets()) { +// String key = bucket.getKey().toString(); +// long count = ((ValueCount) bucket.getAggregations().get("COUNT(*)")).getValue(); +// if(key.equalsIgnoreCase("m")) { +// Assert.assertEquals(507, count); +// } +// else { +// throw new Exception(String.format("Unexpected key. expected: only m. found: %s", key)); +// } +// } +// } +// +// + //endregion not migrated + + @Test + public void groupByOnNestedFieldWithFilterTest() throws Exception { + + String query = String.format("SELECT COUNT(*) FROM %s GROUP BY nested(message.info)," + + "filter('myFilter',message.info = 'a')", TEST_INDEX_NESTED_TYPE); + JSONObject result = executeQuery(query); + + JSONObject aggregation = getAggregation(result, "message.info@NESTED"); + JSONArray buckets = (JSONArray) aggregation.optQuery("/myFilter@FILTER/message.info/buckets"); + Assert.assertNotNull(buckets); + Assert.assertThat(buckets.length(), equalTo(1)); + + JSONObject bucket = buckets.getJSONObject(0); + Assert.assertThat(bucket.getString("key"), equalTo("a")); + Assert.assertThat(bucket.query("/COUNT(*)/value"), equalTo(2)); + } + + @Test + public void minOnNestedField() throws Exception { + + String query = String.format("SELECT min(nested(message.dayOfWeek)) as minDays FROM %s", + TEST_INDEX_NESTED_TYPE); + JSONObject result = executeQuery(query); + JSONObject aggregation = getAggregation(result, "message.dayOfWeek@NESTED"); + Assert.assertEquals(1.0, (double) aggregation.query("/minDays/value"), 0.0001); + } + + @Test + public void sumOnNestedField() throws Exception { + + String query = String.format("SELECT sum(nested(message.dayOfWeek)) as sumDays FROM %s", + TEST_INDEX_NESTED_TYPE); + JSONObject result = executeQuery(query); + JSONObject aggregation = getAggregation(result, "message.dayOfWeek@NESTED"); + Assert.assertEquals(19.0, (double) aggregation.query("/sumDays/value"), 0.0001); + } + + @Test + public void histogramOnNestedField() throws Exception { + + String query = String.format("select count(*) from %s group by histogram" + + "('field'='message.dayOfWeek','nested'='message','interval'='2' , 'alias' = 'someAlias' )", + TEST_INDEX_NESTED_TYPE); + JSONObject result = executeQuery(query); + JSONObject aggregation = getAggregation(result, "message@NESTED"); + + final Map expectedCountsByKey = new HashMap<>(); + expectedCountsByKey.put(0.0, 2); + expectedCountsByKey.put(2.0, 1); + expectedCountsByKey.put(4.0, 2); + expectedCountsByKey.put(6.0, 1); + + JSONArray buckets = (JSONArray) aggregation.query("/someAlias/buckets"); + Assert.assertThat(buckets.length(), equalTo(4)); + + buckets.forEach(obj -> { + JSONObject bucket = (JSONObject) obj; + final double key = bucket.getDouble("key"); + Assert.assertTrue(expectedCountsByKey.containsKey(key)); + Assert.assertThat(bucket.getJSONObject("COUNT(*)").getInt("value"), + equalTo(expectedCountsByKey.get(key))); + }); + } + + @Test + public void reverseToRootGroupByOnNestedFieldWithFilterTestWithReverseNestedAndEmptyPath() + throws Exception { + + String query = String.format("SELECT COUNT(*) FROM %s GROUP BY nested(message.info)," + + "filter('myFilter',message.info = 'a'),reverse_nested(someField,'')", + TEST_INDEX_NESTED_TYPE); + JSONObject result = executeQuery(query); + JSONObject aggregation = getAggregation(result, "message.info@NESTED"); + + JSONArray msgInfoBuckets = + (JSONArray) aggregation.optQuery("/myFilter@FILTER/message.info/buckets"); + Assert.assertNotNull(msgInfoBuckets); + Assert.assertThat(msgInfoBuckets.length(), equalTo(1)); + + JSONArray someFieldBuckets = + (JSONArray) msgInfoBuckets.optQuery("/0/someField@NESTED/someField/buckets"); + Assert.assertNotNull(msgInfoBuckets); + Assert.assertThat(msgInfoBuckets.length(), equalTo(1)); + Assert.assertThat(someFieldBuckets.query("/0/key"), equalTo("b")); + Assert.assertThat(someFieldBuckets.query("/0/COUNT(*)/value"), equalTo(2)); + } + + @Test + public void reverseToRootGroupByOnNestedFieldWithFilterTestWithReverseNestedNoPath() + throws Exception { + + String query = String.format("SELECT COUNT(*) FROM %s GROUP BY nested(message.info),filter" + + "('myFilter',message.info = 'a'),reverse_nested(someField)", TEST_INDEX_NESTED_TYPE); + JSONObject result = executeQuery(query); + JSONObject aggregation = getAggregation(result, "message.info@NESTED"); + + JSONArray msgInfoBuckets = + (JSONArray) aggregation.optQuery("/myFilter@FILTER/message.info/buckets"); + Assert.assertNotNull(msgInfoBuckets); + Assert.assertThat(msgInfoBuckets.length(), equalTo(1)); + + JSONArray someFieldBuckets = + (JSONArray) msgInfoBuckets.optQuery("/0/someField@NESTED/someField/buckets"); + Assert.assertNotNull(msgInfoBuckets); + Assert.assertThat(someFieldBuckets.length(), equalTo(1)); + Assert.assertThat(someFieldBuckets.query("/0/key"), equalTo("b")); + Assert.assertThat(someFieldBuckets.query("/0/COUNT(*)/value"), equalTo(2)); + } + + @Test + public void reverseToRootGroupByOnNestedFieldWithFilterTestWithReverseNestedOnHistogram() + throws Exception { + + String query = String.format("SELECT COUNT(*) FROM %s GROUP BY nested(message.info)," + + "filter('myFilter',message.info = 'a'),histogram('field'='myNum','reverse_nested'='','interval'='2', " + + "'alias' = 'someAlias' )", TEST_INDEX_NESTED_TYPE); + JSONObject result = executeQuery(query); + JSONObject aggregation = getAggregation(result, "message.info@NESTED"); + + JSONArray msgInfoBuckets = + (JSONArray) aggregation.optQuery("/myFilter@FILTER/message.info/buckets"); + Assert.assertNotNull(msgInfoBuckets); + Assert.assertThat(msgInfoBuckets.length(), equalTo(1)); + + JSONArray someAliasBuckets = + (JSONArray) msgInfoBuckets.optQuery("/0/someAlias@NESTED/someAlias/buckets"); + Assert.assertNotNull(msgInfoBuckets); + Assert.assertThat(someAliasBuckets.length(), equalTo(3)); + + final Map expectedCountsByKey = new HashMap<>(); + expectedCountsByKey.put(0.0, 1); + expectedCountsByKey.put(2.0, 0); + expectedCountsByKey.put(4.0, 1); + + someAliasBuckets.forEach(obj -> { + JSONObject bucket = (JSONObject) obj; + final double key = bucket.getDouble("key"); + Assert.assertTrue(expectedCountsByKey.containsKey(key)); + Assert.assertThat(bucket.getJSONObject("COUNT(*)").getInt("value"), + equalTo(expectedCountsByKey.get(key))); + }); + } + + @Test + public void reverseToRootGroupByOnNestedFieldWithFilterAndSumOnReverseNestedField() + throws Exception { + + String query = String.format("SELECT sum(reverse_nested(myNum)) bla FROM %s GROUP BY " + + "nested(message.info),filter('myFilter',message.info = 'a')", TEST_INDEX_NESTED_TYPE); + JSONObject result = executeQuery(query); + JSONObject aggregation = getAggregation(result, "message.info@NESTED"); + + JSONArray msgInfoBuckets = + (JSONArray) aggregation.optQuery("/myFilter@FILTER/message.info/buckets"); + Assert.assertNotNull(msgInfoBuckets); + Assert.assertThat(msgInfoBuckets.length(), equalTo(1)); + + Assert.assertNotNull(msgInfoBuckets.optQuery("/0/myNum@NESTED/bla/value")); + JSONObject bla = (JSONObject) msgInfoBuckets.query("/0/myNum@NESTED/bla"); + Assert.assertEquals(5.0, bla.getDouble("value"), 0.000001); + } + + @Test + public void reverseAnotherNestedGroupByOnNestedFieldWithFilterTestWithReverseNestedNoPath() + throws Exception { + + String query = String.format("SELECT COUNT(*) FROM %s GROUP BY nested(message.info)," + + "filter('myFilter',message.info = 'a'),reverse_nested(comment.data,'~comment')", + TEST_INDEX_NESTED_TYPE); + JSONObject result = executeQuery(query); + JSONObject aggregation = getAggregation(result, "message.info@NESTED"); + + JSONArray msgInfoBuckets = + (JSONArray) aggregation.optQuery("/myFilter@FILTER/message.info/buckets"); + Assert.assertNotNull(msgInfoBuckets); + Assert.assertThat(msgInfoBuckets.length(), equalTo(1)); + + JSONArray commentDataBuckets = + (JSONArray) msgInfoBuckets.optQuery("/0/comment.data@NESTED_REVERSED" + + "/comment.data@NESTED/comment.data/buckets"); + Assert.assertNotNull(commentDataBuckets); + Assert.assertThat(commentDataBuckets.length(), equalTo(1)); + Assert.assertThat(commentDataBuckets.query("/0/key"), equalTo("ab")); + Assert.assertThat(commentDataBuckets.query("/0/COUNT(*)/value"), equalTo(2)); + } + + @Test + public void reverseAnotherNestedGroupByOnNestedFieldWithFilterTestWithReverseNestedOnHistogram() + throws Exception { + + String query = String.format("SELECT COUNT(*) FROM %s GROUP BY nested(message.info),filter" + + "('myFilter',message.info = 'a'),histogram('field'='comment.likes','reverse_nested'='~comment'," + + "'interval'='2' , 'alias' = 'someAlias' )", TEST_INDEX_NESTED_TYPE); + JSONObject result = executeQuery(query); + JSONObject aggregation = getAggregation(result, "message.info@NESTED"); + + JSONArray msgInfoBuckets = + (JSONArray) aggregation.optQuery("/myFilter@FILTER/message.info/buckets"); + Assert.assertNotNull(msgInfoBuckets); + Assert.assertThat(msgInfoBuckets.length(), equalTo(1)); + + JSONArray someAliasBuckets = (JSONArray) msgInfoBuckets.optQuery( + "/0/~comment@NESTED_REVERSED/~comment@NESTED/someAlias/buckets"); + Assert.assertNotNull(msgInfoBuckets); + Assert.assertThat(someAliasBuckets.length(), equalTo(2)); + + final Map expectedCountsByKey = new HashMap<>(); + expectedCountsByKey.put(0.0, 1); + expectedCountsByKey.put(2.0, 1); + + someAliasBuckets.forEach(obj -> { + JSONObject bucket = (JSONObject) obj; + final double key = bucket.getDouble("key"); + Assert.assertTrue(expectedCountsByKey.containsKey(key)); + Assert.assertThat(bucket.getJSONObject("COUNT(*)").getInt("value"), + equalTo(expectedCountsByKey.get(key))); + }); + } + + @Test + public void reverseAnotherNestedGroupByOnNestedFieldWithFilterAndSumOnReverseNestedField() + throws Exception { + + String query = + String.format("SELECT sum(reverse_nested(comment.likes,'~comment')) bla FROM %s " + + "GROUP BY nested(message.info),filter('myFilter',message.info = 'a')", + TEST_INDEX_NESTED_TYPE); + JSONObject result = executeQuery(query); + JSONObject aggregation = getAggregation(result, "message.info@NESTED"); + + JSONArray msgInfoBuckets = + (JSONArray) aggregation.optQuery("/myFilter@FILTER/message.info/buckets"); + Assert.assertNotNull(msgInfoBuckets); + Assert.assertThat(msgInfoBuckets.length(), equalTo(1)); + + Assert.assertNotNull(msgInfoBuckets.optQuery( + "/0/comment.likes@NESTED_REVERSED/comment.likes@NESTED/bla/value")); + JSONObject bla = (JSONObject) msgInfoBuckets + .query("/0/comment.likes@NESTED_REVERSED/comment.likes@NESTED/bla"); + Assert.assertEquals(4.0, bla.getDouble("value"), 0.000001); + } + + @Test + public void docsReturnedTestWithoutDocsHint() throws Exception { + String query = String.format("SELECT count(*) from %s", TEST_INDEX_ACCOUNT); + JSONObject result = executeQuery(query); + Assert.assertThat(getHits(result).length(), equalTo(0)); + } + + @Test + public void docsReturnedTestWithDocsHint() throws Exception { + String query = String.format("SELECT /*! DOCS_WITH_AGGREGATION(10) */ count(*) from %s", + TEST_INDEX_ACCOUNT); + JSONObject result = executeQuery(query); + Assert.assertThat(getHits(result).length(), equalTo(10)); + } + + @Ignore("There is not any text field in the index. Need fix later") + @Test + public void termsWithScript() throws Exception { + String query = + String.format("select count(*), avg(all_client) from %s group by terms('alias'='asdf'," + + " substring(field, 0, 1)), date_histogram('alias'='time', 'field'='timestamp', " + + "'interval'='20d ', 'format'='yyyy-MM-dd') limit 1000", TEST_INDEX_ONLINE); + String result = explainQuery(query); + + Assert.assertThat(result, containsString("\"script\":{\"source\"")); + Assert.assertThat(result, containsString("substring(0, 1)")); + } + + @Test + public void groupByScriptedDateHistogram() throws Exception { + String query = String + .format("select count(*), avg(all_client) from %s group by date_histogram('alias'='time'," + + " ceil(all_client), 'fixed_interval'='20d ', 'format'='yyyy-MM-dd') limit 1000", + TEST_INDEX_ONLINE); + String result = explainQuery(query); + + Assert.assertThat(result, containsString("Math.ceil(doc['all_client'].value);")); + Assert.assertThat(result, containsString("\"script\":{\"source\"")); + } + + @Test + public void groupByScriptedHistogram() throws Exception { + String query = String.format( + "select count(*) from %s group by histogram('alias'='all_field', pow(all_client,1))", + TEST_INDEX_ONLINE); + String result = explainQuery(query); + + Assert.assertThat(result, containsString("Math.pow(doc['all_client'].value, 1)")); + Assert.assertThat(result, containsString("\"script\":{\"source\"")); + } + + @Test + public void distinctWithOneField() { + Assert.assertEquals( + executeQuery("SELECT DISTINCT name.lastname FROM " + TEST_INDEX_GAME_OF_THRONES, "jdbc"), + executeQuery("SELECT name.lastname FROM " + TEST_INDEX_GAME_OF_THRONES + + " GROUP BY name.lastname", "jdbc") + ); + } + + @Test + public void distinctWithMultipleFields() { + Assert.assertEquals( + executeQuery("SELECT DISTINCT age, gender FROM " + TEST_INDEX_ACCOUNT, "jdbc"), + executeQuery("SELECT age, gender FROM " + TEST_INDEX_ACCOUNT + + " GROUP BY age, gender", "jdbc") + ); + } + + private JSONObject getAggregation(final JSONObject queryResult, final String aggregationName) { + final String aggregationsObjName = "aggregations"; + Assert.assertTrue(queryResult.has(aggregationsObjName)); + + final JSONObject aggregations = queryResult.getJSONObject(aggregationsObjName); + Assert.assertTrue(aggregations.has(aggregationName)); + return aggregations.getJSONObject(aggregationName); + } + + private int getIntAggregationValue(final JSONObject queryResult, final String aggregationName, + final String fieldName) { + + final JSONObject targetAggregation = getAggregation(queryResult, aggregationName); + Assert.assertTrue(targetAggregation.has(fieldName)); + return targetAggregation.getInt(fieldName); + } + + private double getDoubleAggregationValue(final JSONObject queryResult, + final String aggregationName, + final String fieldName) { + + final JSONObject targetAggregation = getAggregation(queryResult, aggregationName); + Assert.assertTrue(targetAggregation.has(fieldName)); + return targetAggregation.getDouble(fieldName); + } + + private double getDoubleAggregationValue(final JSONObject queryResult, + final String aggregationName, + final String fieldName, final String subFieldName) { + + final JSONObject targetAggregation = getAggregation(queryResult, aggregationName); + Assert.assertTrue(targetAggregation.has(fieldName)); + final JSONObject targetField = targetAggregation.getJSONObject(fieldName); + Assert.assertTrue(targetField.has(subFieldName)); + + return targetField.getDouble(subFieldName); + } + + private JSONObject executeJdbcRequest(String query) { + return new JSONObject(executeQuery(query, "jdbc")); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/CsvFormatResponseIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/CsvFormatResponseIT.java new file mode 100644 index 0000000000..2a9dfd15c2 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/CsvFormatResponseIT.java @@ -0,0 +1,795 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_DOG; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_GAME_OF_THRONES; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_TYPE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_WITH_QUOTES; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ONLINE; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.core.StringStartsWith.startsWith; + +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.csv.CSVResult; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.hamcrest.Matcher; +import org.hamcrest.core.AnyOf; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + +/** + * Tests to cover requests with "?format=csv" parameter + */ +public class CsvFormatResponseIT extends SQLIntegTestCase { + + private boolean flatOption = false; + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + loadIndex(Index.NESTED); + loadIndex(Index.NESTED_WITH_QUOTES); + loadIndex(Index.DOG); + loadIndex(Index.GAME_OF_THRONES); + loadIndex(Index.ONLINE); + } + + @Override + protected Request getSqlRequest(String request, boolean explain) { + + Request sqlRequest = super.getSqlRequest(request, explain); + sqlRequest.addParameter("format", "csv"); + sqlRequest.addParameter("flat", flatOption ? "true" : "false"); + return sqlRequest; + } + + @Test + public void allPercentilesByDefault() throws IOException { + + final String query = + String.format(Locale.ROOT, "SELECT PERCENTILES(age) FROM %s", TEST_INDEX_ACCOUNT); + final String result = executeQueryWithStringOutput(query); + + final String expectedHeaders = + "PERCENTILES(age).1.0,PERCENTILES(age).5.0,PERCENTILES(age).25.0," + + "PERCENTILES(age).50.0,PERCENTILES(age).75.0,PERCENTILES(age).95.0,PERCENTILES(age).99.0"; + Assert.assertThat(result, containsString(expectedHeaders)); + } + + @Test + public void specificPercentilesIntAndDouble() throws IOException { + + final String query = String.format(Locale.ROOT, "SELECT PERCENTILES(age,10,49.0) FROM %s", + TEST_INDEX_ACCOUNT); + final String result = executeQueryWithStringOutput(query); + + final String[] unexpectedPercentiles = {"1.0", "5.0", "25.0", "50.0", "75.0", "95.0", "99.0"}; + final String expectedHeaders = + "\"PERCENTILES(age,10,49.0).10.0\",\"PERCENTILES(age,10,49.0).49.0\""; + Assert.assertThat(result, containsString(expectedHeaders)); + for (final String unexpectedPercentile : unexpectedPercentiles) { + Assert.assertThat(result, + not(containsString("PERCENTILES(age,10,49.0)." + unexpectedPercentile))); + } + } + + @Test + public void nestedObjectsAndArraysAreQuoted() throws IOException { + + final String query = String.format(Locale.ROOT, "SELECT * FROM %s WHERE _id = 5", + TEST_INDEX_NESTED_TYPE); + final String result = executeQueryWithStringOutput(query); + + final String expectedMyNum = "\"[3, 4]\""; + final String expectedComment = "\"{data=[aa, bb], likes=10}\""; + final String expectedMessage = "\"[{dayOfWeek=6, author=zz, info=zz}]\""; + + Assert.assertThat(result, containsString(expectedMyNum)); + Assert.assertThat(result, containsString(expectedComment)); + Assert.assertThat(result, containsString(expectedMessage)); + } + + @Test + public void arraysAreQuotedInFlatMode() throws IOException { + + setFlatOption(true); + + final String query = String.format(Locale.ROOT, "SELECT * FROM %s WHERE _id = 5", + TEST_INDEX_NESTED_TYPE); + final String result = executeQueryWithStringOutput(query); + + final String expectedMyNum = "\"[3, 4]\""; + final String expectedCommentData = "\"[aa, bb]\""; + final String expectedMessage = "\"[{dayOfWeek=6, author=zz, info=zz}]\""; + + Assert.assertThat(result, containsString(expectedMyNum)); + Assert.assertThat(result, containsString(expectedCommentData)); + Assert.assertThat(result, containsString(expectedMessage)); + + setFlatOption(false); + } + + @Test + public void doubleQuotesAreEscapedWithDoubleQuotes() throws IOException { + final String query = "SELECT * FROM " + TEST_INDEX_NESTED_WITH_QUOTES; + + final CSVResult csvResult = executeCsvRequest(query, false); + final List rows = csvResult.getLines(); + Assert.assertThat(rows.size(), equalTo(2)); + + final String expectedValue1 = "\"[{dayOfWeek=6, author=z\"\"z, info=zz}]\""; + final String expectedValue2 = + "\"[{dayOfWeek=3, author=this \"\"value\"\" contains quotes, info=rr}]\""; + + for (String row : rows) { + Assert.assertThat(row, anyOf(containsString(expectedValue1), containsString(expectedValue2))); + } + } + + @Test + public void fieldOrder() throws IOException { + + final String[] expectedFields = {"age", "firstname", "address", "gender", "email"}; + + verifyFieldOrder(expectedFields); + } + + @Test + public void fieldOrderOther() throws IOException { + + final String[] expectedFields = {"email", "firstname", "age", "gender", "address"}; + + verifyFieldOrder(expectedFields); + } + + @Ignore("Getting parser error") + @Test + public void fieldOrderWithScriptFields() throws IOException { + + final String[] expectedFields = {"email", "script1", "script2", "gender", "address"}; + final String query = String.format(Locale.ROOT, "SELECT email, " + + "script(script1, \"doc['balance'].value * 2\"), " + + "script(script2, painless, \"doc['balance'].value + 10\"), gender, address " + + "FROM %s WHERE email='amberduke@pyrami.com'", TEST_INDEX_ACCOUNT); + + verifyFieldOrder(expectedFields, query); + } + + //region Tests migrated from CSVResultsExtractorTests + + @Test + public void simpleSearchResultNotNestedNotFlatNoAggs() throws Exception { + String query = + String.format(Locale.ROOT, "select dog_name,age from %s order by age", TEST_INDEX_DOG); + final CSVResult csvResult = executeCsvRequest(query, false); + + List headers = csvResult.getHeaders(); + Assert.assertEquals(2, headers.size()); + Assert.assertTrue("name should be on headers", headers.contains("dog_name")); + Assert.assertTrue("age should be on headers", headers.contains("age")); + + List lines = csvResult.getLines(); + Assert.assertEquals(2, lines.size()); + Assert.assertTrue("rex,2".equals(lines.get(0)) || "2,rex".equals(lines.get(0))); + Assert.assertTrue("snoopy,4".equals(lines.get(1)) || "4,snoopy".equals(lines.get(1))); + } + + @Test + public void simpleSearchResultWithNestedNotFlatNoAggs() throws Exception { + String query = String.format(Locale.ROOT, "select name,house from %s", + TEST_INDEX_GAME_OF_THRONES); + CSVResult csvResult = executeCsvRequest(query, false); + + List headers = csvResult.getHeaders(); + Assert.assertEquals(2, headers.size()); + Assert.assertTrue("name should be on headers", headers.contains("name")); + Assert.assertTrue("house should be on headers", headers.contains("house")); + + List lines = csvResult.getLines(); + Assert.assertEquals(7, lines.size()); + + Assert.assertThat(lines, hasRow(null, "Targaryen", + Arrays.asList("firstname=Daenerys", "lastname=Targaryen", "ofHerName=1"), true)); + Assert.assertThat(lines, hasRow(null, "Stark", + Arrays.asList("firstname=Eddard", "lastname=Stark", "ofHisName=1"), true)); + Assert.assertThat(lines, hasRow(null, "Stark", + Arrays.asList("firstname=Brandon", "lastname=Stark", "ofHisName=4"), true)); + Assert.assertThat(lines, hasRow(null, "Lannister", + Arrays.asList("firstname=Jaime", "lastname=Lannister", "ofHisName=1"), true)); + } + + @Ignore("headers incorrect in case of nested fields") + @Test + public void simpleSearchResultWithNestedOneFieldNotFlatNoAggs() throws Exception { + String query = String.format(Locale.ROOT, "select name.firstname,house from %s", + TEST_INDEX_GAME_OF_THRONES); + CSVResult csvResult = executeCsvRequest(query, false); + + List headers = csvResult.getHeaders(); + Assert.assertEquals(2, headers.size()); + Assert.assertTrue("name should be on headers", headers.contains("name")); + Assert.assertTrue("house should be on headers", headers.contains("house")); + + List lines = csvResult.getLines(); + Assert.assertEquals(7, lines.size()); + Assert.assertThat(lines, hasItem("{firstname=Daenerys},Targaryen")); + Assert.assertThat(lines, hasItem("{firstname=Eddard},Stark")); + Assert.assertThat(lines, hasItem("{firstname=Brandon},Stark")); + Assert.assertThat(lines, hasItem("{firstname=Jaime},Lannister")); + + } + + @Ignore("headers incorrect in case of nested fields") + @Test + public void simpleSearchResultWithNestedTwoFieldsFromSameNestedNotFlatNoAggs() throws Exception { + String query = String.format(Locale.ROOT, "select name.firstname,name.lastname,house from %s", + TEST_INDEX_GAME_OF_THRONES); + CSVResult csvResult = executeCsvRequest(query, false); + + List headers = csvResult.getHeaders(); + Assert.assertEquals(2, headers.size()); + Assert.assertTrue("name should be on headers", headers.contains("name")); + Assert.assertTrue("house should be on headers", headers.contains("house")); + + List lines = csvResult.getLines(); + Assert.assertEquals(7, lines.size()); + + Assert.assertThat(lines, hasRow(null, "Targaryen", + Arrays.asList("firstname=Daenerys", "lastname=Targaryen"), true)); + Assert.assertThat(lines, hasRow(null, "Stark", + Arrays.asList("firstname=Eddard", "lastname=Stark"), true)); + Assert.assertThat(lines, hasRow(null, "Stark", + Arrays.asList("firstname=Brandon", "lastname=Stark"), true)); + Assert.assertThat(lines, hasRow(null, "Lannister", + Arrays.asList("firstname=Jaime", "lastname=Lannister"), true)); + } + + @Test + public void simpleSearchResultWithNestedWithFlatNoAggs() throws Exception { + String query = String.format(Locale.ROOT, "select name.firstname,house from %s", + TEST_INDEX_GAME_OF_THRONES); + CSVResult csvResult = executeCsvRequest(query, true); + + List headers = csvResult.getHeaders(); + Assert.assertEquals(2, headers.size()); + Assert.assertTrue("name.firstname should be on headers", headers.contains("name.firstname")); + Assert.assertTrue("house should be on headers", headers.contains("house")); + + List lines = csvResult.getLines(); + Assert.assertEquals(7, lines.size()); + Assert.assertTrue(lines.contains("Daenerys,Targaryen")); + Assert.assertTrue(lines.contains("Eddard,Stark")); + Assert.assertTrue(lines.contains("Brandon,Stark")); + Assert.assertTrue(lines.contains("Jaime,Lannister")); + } + + @Test + public void joinSearchResultNotNestedNotFlatNoAggs() throws Exception { + String query = String.format(Locale.ROOT, "select c.gender , h.hname,h.words from %s c " + + "JOIN %s h " + + "on h.hname = c.house ", TEST_INDEX_GAME_OF_THRONES, TEST_INDEX_GAME_OF_THRONES); + CSVResult csvResult = executeCsvRequest(query, false); + + List headers = csvResult.getHeaders(); + Assert.assertEquals(3, headers.size()); + Assert.assertTrue("c.gender should be on headers", headers.contains("c.gender")); + Assert.assertTrue("h.hname should be on headers", headers.contains("h.hname")); + Assert.assertTrue("h.words should be on headers", headers.contains("h.words")); + + List lines = csvResult.getLines(); + Assert.assertEquals(4, lines.size()); + + Assert.assertThat(lines, + hasRow(null, null, Arrays.asList("F", "fireAndBlood", "Targaryen"), false)); + } + + @Test + public void simpleNumericValueAgg() throws Exception { + String query = String.format(Locale.ROOT, "select count(*) from %s ", TEST_INDEX_DOG); + CSVResult csvResult = executeCsvRequest(query, false); + + List headers = csvResult.getHeaders(); + Assert.assertEquals(1, headers.size()); + Assert.assertEquals("COUNT(*)", headers.get(0)); + + + List lines = csvResult.getLines(); + Assert.assertEquals(1, lines.size()); + Assert.assertEquals("2.0", lines.get(0)); + + } + + @Test + public void simpleNumericValueAggWithAlias() throws Exception { + String query = + String.format(Locale.ROOT, "select avg(age) as myAlias from %s ", TEST_INDEX_DOG); + CSVResult csvResult = executeCsvRequest(query, false); + + List headers = csvResult.getHeaders(); + Assert.assertEquals(1, headers.size()); + Assert.assertEquals("myAlias", headers.get(0)); + + + List lines = csvResult.getLines(); + Assert.assertEquals(1, lines.size()); + Assert.assertEquals("3.0", lines.get(0)); + + } + + @Test + public void twoNumericAggWithAlias() throws Exception { + String query = + String.format(Locale.ROOT, "select count(*) as count, avg(age) as myAlias from %s ", + TEST_INDEX_DOG); + CSVResult csvResult = executeCsvRequest(query, false); + + List headers = csvResult.getHeaders(); + Assert.assertEquals(2, headers.size()); + + Assert.assertTrue(headers.contains("count")); + Assert.assertTrue(headers.contains("myAlias")); + + + List lines = csvResult.getLines(); + Assert.assertEquals(1, lines.size()); + if (headers.get(0).equals("count")) { + Assert.assertEquals("2.0,3.0", lines.get(0)); + } else { + Assert.assertEquals("3.0,2.0", lines.get(0)); + } + + } + + @Test + public void aggAfterTermsGroupBy() throws Exception { + String query = String.format(Locale.ROOT, "SELECT COUNT(*) FROM %s GROUP BY gender", + TEST_INDEX_ACCOUNT); + CSVResult csvResult = executeCsvRequest(query, false); + List headers = csvResult.getHeaders(); + Assert.assertEquals(1, headers.size()); + assertThat(headers, contains(equalTo("COUNT(*)"))); + + List lines = csvResult.getLines(); + Assert.assertEquals(2, lines.size()); + assertThat(lines, containsInAnyOrder(equalTo("507.0"), equalTo("493.0"))); + } + + @Test + public void aggAfterTwoTermsGroupBy() throws Exception { + String query = String.format(Locale.ROOT, + "SELECT COUNT(*) FROM %s where age in (35,36) GROUP BY gender,age", + TEST_INDEX_ACCOUNT); + CSVResult csvResult = executeCsvRequest(query, false); + List headers = csvResult.getHeaders(); + Assert.assertEquals(1, headers.size()); + assertThat(headers, contains(equalTo("COUNT(*)"))); + + List lines = csvResult.getLines(); + Assert.assertEquals(4, lines.size()); + assertThat(lines, containsInAnyOrder( + equalTo("31.0"), + equalTo("28.0"), + equalTo("21.0"), + equalTo("24.0"))); + } + + @Test + public void multipleAggAfterTwoTermsGroupBy() throws Exception { + String query = String.format(Locale.ROOT, + "SELECT COUNT(*) , sum(balance) FROM %s where age in (35,36) GROUP BY gender,age", + TEST_INDEX_ACCOUNT); + CSVResult csvResult = executeCsvRequest(query, false); + List headers = csvResult.getHeaders(); + Assert.assertEquals(2, headers.size()); + assertThat(headers, contains(equalTo("COUNT(*)"), equalTo("SUM(balance)"))); + + List lines = csvResult.getLines(); + Assert.assertEquals(4, lines.size()); + assertThat(lines, containsInAnyOrder( + equalTo("31.0,647425.0"), + equalTo("28.0,678337.0"), + equalTo("21.0,505660.0"), + equalTo("24.0,472771.0"))); + } + + @Test + public void dateHistogramTest() throws Exception { + String query = String.format(Locale.ROOT, "select count(*) from %s" + + " group by date_histogram('field'='insert_time','fixed_interval'='4d','alias'='days')", + TEST_INDEX_ONLINE); + CSVResult csvResult = executeCsvRequest(query, false); + List headers = csvResult.getHeaders(); + Assert.assertEquals(1, headers.size()); + assertThat(headers, contains(equalTo("COUNT(*)"))); + + List lines = csvResult.getLines(); + Assert.assertEquals(3, lines.size()); + assertThat(lines, containsInAnyOrder( + equalTo("477.0"), + equalTo("5664.0"), + equalTo("3795.0"))); + } + + @Test + public void statsAggregationTest() throws Exception { + String query = String.format(Locale.ROOT, "SELECT STATS(age) FROM %s", TEST_INDEX_ACCOUNT); + CSVResult csvResult = executeCsvRequest(query, false); + List headers = csvResult.getHeaders(); + Assert.assertEquals(5, headers.size()); + Assert.assertEquals("STATS(age).count", headers.get(0)); + Assert.assertEquals("STATS(age).sum", headers.get(1)); + Assert.assertEquals("STATS(age).avg", headers.get(2)); + Assert.assertEquals("STATS(age).min", headers.get(3)); + Assert.assertEquals("STATS(age).max", headers.get(4)); + + List lines = csvResult.getLines(); + Assert.assertEquals(1, lines.size()); + Assert.assertEquals("1000,30171.0,30.171,20.0,40.0", lines.get(0)); + } + + @Test + public void extendedStatsAggregationTest() throws Exception { + String query = + String.format(Locale.ROOT, "SELECT EXTENDED_STATS(age) FROM %s", TEST_INDEX_ACCOUNT); + CSVResult csvResult = executeCsvRequest(query, false); + List headers = csvResult.getHeaders(); + + final String[] expectedHeaders = {"EXTENDED_STATS(age).count", "EXTENDED_STATS(age).sum", + "EXTENDED_STATS(age).avg", "EXTENDED_STATS(age).min", "EXTENDED_STATS(age).max", + "EXTENDED_STATS(age).sumOfSquares", "EXTENDED_STATS(age).variance", + "EXTENDED_STATS(age).stdDeviation"}; + + Assert.assertEquals(expectedHeaders.length, headers.size()); + Assert.assertThat(headers, contains(expectedHeaders)); + + List lines = csvResult.getLines(); + Assert.assertEquals(1, lines.size()); + String line = lines.get(0); + Assert.assertTrue(line.startsWith("1000,30171.0,30.171,20.0,40.0,946393.0")); + Assert.assertTrue(line.contains(",6.008")); + Assert.assertTrue(line.contains(",36.103")); + } + + @Test + public void percentileAggregationTest() throws Exception { + String query = + String.format(Locale.ROOT, "select percentiles(age) as per from %s where age > 31", + TEST_INDEX_ACCOUNT); + CSVResult csvResult = executeCsvRequest(query, false); + List headers = csvResult.getHeaders(); + Assert.assertEquals(7, headers.size()); + Assert.assertEquals("per.1.0", headers.get(0)); + Assert.assertEquals("per.5.0", headers.get(1)); + Assert.assertEquals("per.25.0", headers.get(2)); + Assert.assertEquals("per.50.0", headers.get(3)); + Assert.assertEquals("per.75.0", headers.get(4)); + Assert.assertEquals("per.95.0", headers.get(5)); + Assert.assertEquals("per.99.0", headers.get(6)); + + + List lines = csvResult.getLines(); + Assert.assertEquals(1, lines.size()); + Assert.assertEquals("32.0,32.0,34.0,36.0,38.0,40.0,40.0", lines.get(0)); + } + + @Test + public void includeTypeAndNotScore() throws Exception { + String query = + String.format(Locale.ROOT, "select age , firstname from %s where age > 31 limit 2", + TEST_INDEX_ACCOUNT); + CSVResult csvResult = executeCsvRequest(query, false, false, true, false); + List headers = csvResult.getHeaders(); + Assert.assertEquals(3, headers.size()); + Assert.assertTrue(headers.contains("age")); + Assert.assertTrue(headers.contains("firstname")); + Assert.assertTrue(headers.contains("_type")); + List lines = csvResult.getLines(); + Assert.assertTrue(lines.get(0).contains(",_doc") || lines.get(0).contains("_doc,")); + Assert.assertTrue(lines.get(1).contains(",_doc") || lines.get(1).contains("_doc,")); + } + + @Test + public void includeScoreAndNotType() throws Exception { + String query = String.format(Locale.ROOT, + "select age , firstname from %s where age > 31 order by _score desc limit 2 ", + TEST_INDEX_ACCOUNT); + CSVResult csvResult = executeCsvRequest(query, false, true, false, false); + List headers = csvResult.getHeaders(); + Assert.assertEquals(3, headers.size()); + Assert.assertTrue(headers.contains("age")); + Assert.assertTrue(headers.contains("firstname")); + Assert.assertTrue(headers.contains("_score")); + List lines = csvResult.getLines(); + Assert.assertTrue(lines.get(0).contains("1.0")); + Assert.assertTrue(lines.get(1).contains("1.0")); + } + + @Test + public void includeScoreAndType() throws Exception { + String query = String.format(Locale.ROOT, + "select age , firstname from %s where age > 31 order by _score desc limit 2 ", + TEST_INDEX_ACCOUNT); + CSVResult csvResult = executeCsvRequest(query, false, true, true, false); + List headers = csvResult.getHeaders(); + Assert.assertEquals(4, headers.size()); + Assert.assertTrue(headers.contains("age")); + Assert.assertTrue(headers.contains("firstname")); + Assert.assertTrue(headers.contains("_score")); + Assert.assertTrue(headers.contains("_type")); + List lines = csvResult.getLines(); + String firstLine = lines.get(0); + Assert.assertTrue(firstLine.contains("_doc,1.0") || firstLine.contains("1.0,_doc")); + Assert.assertTrue(lines.get(1).contains("_doc,1.0") || lines.get(1).contains("1.0,_doc")); + } + + /* todo: more tests: + * filter/nested and than metric + * histogram + * geo + */ + + @Test + public void scriptedField() throws Exception { + String query = String.format(Locale.ROOT, + "select age+1 as agePlusOne ,age , firstname from %s where age = 31 limit 1", + TEST_INDEX_ACCOUNT); + CSVResult csvResult = executeCsvRequest(query, false); + List headers = csvResult.getHeaders(); + Assert.assertEquals(3, headers.size()); + Assert.assertTrue(headers.contains("agePlusOne")); + Assert.assertTrue(headers.contains("age")); + Assert.assertTrue(headers.contains("firstname")); + List lines = csvResult.getLines(); + Assert.assertTrue(lines.get(0).contains("32,31") || lines.get(0).contains("32.0,31.0") || + lines.get(0).contains("31,32") || lines.get(0).contains("31.0,32.0")); + } + + @Ignore("separator not exposed") + @Test + public void twoCharsSeperator() throws Exception { + String query = + String.format(Locale.ROOT, "select dog_name,age from %s order by age", TEST_INDEX_DOG); + CSVResult csvResult = executeCsvRequest(query, false); + + List headers = csvResult.getHeaders(); + Assert.assertEquals(2, headers.size()); + Assert.assertTrue("name should be on headers", headers.contains("dog_name")); + Assert.assertTrue("age should be on headers", headers.contains("age")); + + List lines = csvResult.getLines(); + Assert.assertEquals(2, lines.size()); + Assert.assertTrue("rex||2".equals(lines.get(0)) || "2||rex".equals(lines.get(0))); + Assert.assertTrue("snoopy||4".equals(lines.get(1)) || "4||snoopy".equals(lines.get(1))); + + } + + @Test + public void includeIdAndNotTypeOrScore() throws Exception { + String query = String.format(Locale.ROOT, + "select age , firstname from %s where lastname = 'Marquez' ", TEST_INDEX_ACCOUNT); + CSVResult csvResult = executeCsvRequest(query, false, false, false, true); + List headers = csvResult.getHeaders(); + Assert.assertEquals(3, headers.size()); + Assert.assertTrue(headers.contains("age")); + Assert.assertTrue(headers.contains("firstname")); + Assert.assertTrue(headers.contains("_id")); + List lines = csvResult.getLines(); + Assert.assertTrue(lines.get(0).contains(",437") || lines.get(0).contains("437,")); + } + + @Test + public void includeIdAndTypeButNoScore() throws Exception { + String query = String.format(Locale.ROOT, + "select age , firstname from %s where lastname = 'Marquez' ", TEST_INDEX_ACCOUNT); + CSVResult csvResult = executeCsvRequest(query, false, false, true, true); + List headers = csvResult.getHeaders(); + Assert.assertEquals(4, headers.size()); + Assert.assertTrue(headers.contains("age")); + Assert.assertTrue(headers.contains("firstname")); + Assert.assertTrue(headers.contains("_id")); + Assert.assertTrue(headers.contains("_type")); + List lines = csvResult.getLines(); + Assert.assertTrue(lines.get(0).contains("_doc,437") || lines.get(0).contains("437,_doc")); + } + //endregion Tests migrated from CSVResultsExtractorTests + + @Test + public void sensitiveCharacterSanitizeTest() throws IOException { + String requestBody = + "{" + + " \"=cmd|' /C notepad'!_xlbgnm.A1\": \"+cmd|' /C notepad'!_xlbgnm.A1\",\n" + + " \"-cmd|' /C notepad'!_xlbgnm.A1\": \"@cmd|' /C notepad'!_xlbgnm.A1\"\n" + + "}"; + + Request request = new Request("PUT", "/userdata/_doc/1?refresh=true"); + request.setJsonEntity(requestBody); + TestUtils.performRequest(client(), request); + + CSVResult csvResult = executeCsvRequest("SELECT * FROM userdata", false, false, false, false); + List headers = csvResult.getHeaders(); + Assert.assertEquals(2, headers.size()); + Assert.assertTrue(headers.contains("'=cmd|' /C notepad'!_xlbgnm.A1")); + Assert.assertTrue(headers.contains("'-cmd|' /C notepad'!_xlbgnm.A1")); + + List lines = csvResult.getLines(); + Assert.assertEquals(1, lines.size()); + Assert.assertTrue(lines.get(0).contains("'+cmd|' /C notepad'!_xlbgnm.A1")); + Assert.assertTrue(lines.get(0).contains("'@cmd|' /C notepad'!_xlbgnm.A1")); + } + + @Test + public void sensitiveCharacterSanitizeAndQuotedTest() throws IOException { + String requestBody = + "{" + + " \"=cmd|' /C notepad'!_xlbgnm.A1,,\": \",+cmd|' /C notepad'!_xlbgnm.A1\",\n" + + " \",@cmd|' /C notepad'!_xlbgnm.A1\": \"+cmd|' /C notepad,,'!_xlbgnm.A1\",\n" + + " \"-cmd|' /C notepad,,'!_xlbgnm.A1\": \",,,@cmd|' /C notepad'!_xlbgnm.A1\"\n" + + "}"; + + Request request = new Request("PUT", "/userdata2/_doc/1?refresh=true"); + request.setJsonEntity(requestBody); + TestUtils.performRequest(client(), request); + + CSVResult csvResult = executeCsvRequest("SELECT * FROM userdata2", false, false, false, false); + String headers = String.join(",", csvResult.getHeaders()); + Assert.assertTrue(headers.contains("\"'=cmd|' /C notepad'!_xlbgnm.A1,,\"")); + Assert.assertTrue(headers.contains("\",@cmd|' /C notepad'!_xlbgnm.A1\"")); + Assert.assertTrue(headers.contains("\"'-cmd|' /C notepad,,'!_xlbgnm.A1\"")); + + List lines = csvResult.getLines(); + Assert.assertEquals(1, lines.size()); + Assert.assertTrue(lines.get(0).contains("\",+cmd|' /C notepad'!_xlbgnm.A1\"")); + Assert.assertTrue(lines.get(0).contains("\"'+cmd|' /C notepad,,'!_xlbgnm.A1\"")); + Assert.assertTrue(lines.get(0).contains("\",,,@cmd|' /C notepad'!_xlbgnm.A1\"")); + } + + @Test + public void selectFunctionAsFieldTest() throws IOException { + String query = "select log(age) from " + TEST_INDEX_ACCOUNT; + CSVResult result = executeCsvRequest(query, false, false, false, false); + List headers = result.getHeaders(); + Assert.assertEquals(1, headers.size()); + } + + private void verifyFieldOrder(final String[] expectedFields) throws IOException { + + final String fields = String.join(", ", expectedFields); + final String query = String.format(Locale.ROOT, "SELECT %s FROM %s " + + "WHERE email='amberduke@pyrami.com'", fields, TEST_INDEX_ACCOUNT); + + verifyFieldOrder(expectedFields, query); + } + + private void verifyFieldOrder(final String[] expectedFields, final String query) + throws IOException { + + final String result = executeQueryWithStringOutput(query); + + final String expectedHeader = String.join(",", expectedFields); + Assert.assertThat(result, startsWith(expectedHeader)); + } + + private void setFlatOption(boolean flat) { + + this.flatOption = flat; + } + + private CSVResult executeCsvRequest(final String query, boolean flat) throws IOException { + + return executeCsvRequest(query, flat, false, false, false); + } + + private CSVResult executeCsvRequest(final String query, boolean flat, boolean includeScore, + boolean includeType, boolean includeId) throws IOException { + + final String requestBody = super.makeRequest(query); + final String endpoint = String.format(Locale.ROOT, + "/_opendistro/_sql?format=csv&flat=%b&_id=%b&_score=%b&_type=%b", + flat, includeId, includeScore, includeType); + final Request sqlRequest = new Request("POST", endpoint); + sqlRequest.setJsonEntity(requestBody); + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + sqlRequest.setOptions(restOptionsBuilder); + + final Response response = client().performRequest(sqlRequest); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + final String responseString = TestUtils.getResponseBody(response, true); + + return csvResultFromStringResponse(responseString); + } + + private CSVResult csvResultFromStringResponse(final String response) { + + final List rows = new ArrayList<>(); + + final String newLine = String.format(Locale.ROOT, "%n"); + int newLineIndex = response.indexOf(newLine); + + final String headerLine; + + if (-1 == newLineIndex) { + // assuming only headers + headerLine = response.trim(); + } else { + headerLine = response.substring(0, newLineIndex).trim(); + final String rowLines = response.substring(newLineIndex + newLine.length()).trim(); + if (!rowLines.isEmpty()) { + rows.addAll(Arrays.asList(rowLines.split(newLine))); + } + } + + final List headers = new ArrayList<>(Arrays.asList(headerLine.split(","))); + return new CSVResult(headers, rows); + } + + private static AnyOf> hasRow(final String prefix, final String suffix, + final List items, + final boolean areItemsNested) { + + final Collection> permutations = TestUtils.getPermutations(items); + + final List>> matchers = permutations.stream().map(permutation -> { + + final String delimiter = areItemsNested ? ", " : ","; + final String objectField = String.join(delimiter, permutation); + final String row = String.format(Locale.ROOT, "%s%s%s%s%s", + printablePrefix(prefix), areItemsNested ? "\"{" : "", + objectField, areItemsNested ? "}\"" : "", printableSuffix(suffix)); + return hasItem(row); + + }).collect(Collectors.toCollection(LinkedList::new)); + + return anyOf(matchers); + } + + private static String printablePrefix(final String prefix) { + + if (prefix == null || prefix.trim().isEmpty()) { + return ""; + } + + return prefix + ","; + } + + private static String printableSuffix(final String suffix) { + + if (suffix == null || suffix.trim().isEmpty()) { + return ""; + } + + return "," + suffix; + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/CursorIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/CursorIT.java new file mode 100644 index 0000000000..160b41dc3a --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/CursorIT.java @@ -0,0 +1,501 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getResponseBody; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_TIME; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_SIMPLE; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Test; + +public class CursorIT extends SQLIntegTestCase { + + private static final String CURSOR = "cursor"; + private static final String DATAROWS = "datarows"; + private static final String SCHEMA = "schema"; + private static final String JDBC = "jdbc"; + private static final String NEW_LINE = "\n"; + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + enableCursorClusterSetting(); + } + + /** + * Acceptable fetch_size are positive numbers. + * For example 0, 24, 53.0, "110" (parsable string) , "786.23" + * Negative values should throw 400 + */ + @Test + public void invalidNegativeFetchSize() throws IOException { + String query = + StringUtils.format("SELECT firstname, state FROM %s", TestsConstants.TEST_INDEX_ACCOUNT); + Response response = null; + try { + String queryResult = executeFetchQuery(query, -2, JDBC); + } catch (ResponseException ex) { + response = ex.getResponse(); + } + + JSONObject resp = new JSONObject(TestUtils.getResponseBody(response)); + assertThat(resp.getInt("status"), equalTo(400)); + assertThat(resp.query("/error/reason"), equalTo("Invalid SQL query")); + assertThat(resp.query("/error/details"), equalTo("Fetch_size must be greater or equal to 0")); + assertThat(resp.query("/error/type"), equalTo("IllegalArgumentException")); + } + + /** + * Non-numeric fetch_size value should throw 400 + */ + @Test + public void invalidNonNumericFetchSize() throws IOException { + String query = + StringUtils.format("SELECT firstname, state FROM %s", TestsConstants.TEST_INDEX_ACCOUNT); + Response response = null; + try { + String queryResult = executeFetchAsStringQuery(query, "hello world", JDBC); + } catch (ResponseException ex) { + response = ex.getResponse(); + } + + JSONObject resp = new JSONObject(TestUtils.getResponseBody(response)); + assertThat(resp.getInt("status"), equalTo(400)); + assertThat(resp.query("/error/reason"), equalTo("Invalid SQL query")); + assertThat(resp.query("/error/details"), equalTo("Failed to parse field [fetch_size]")); + assertThat(resp.query("/error/type"), equalTo("IllegalArgumentException")); + } + + @Test + public void testExceptionOnCursorExplain() throws IOException { + String cursorRequest = "{\"cursor\":\"d:eyJhIjp7fSwicyI6IkRYRjFaWEo1\"}"; + Request sqlRequest = getSqlRequest(cursorRequest, true); + Response response = null; + try { + String queryResult = executeRequest(sqlRequest); + } catch (ResponseException ex) { + response = ex.getResponse(); + } + + JSONObject resp = new JSONObject(TestUtils.getResponseBody(response)); + assertThat(resp.getInt("status"), equalTo(400)); + assertThat(resp.query("/error/reason"), equalTo("Invalid SQL query")); + assertThat(resp.query("/error/details"), equalTo("Invalid request. Cannot explain cursor")); + assertThat(resp.query("/error/type"), equalTo("IllegalArgumentException")); + } + + /** + * For fetch_size = 0, default to non-pagination behaviour for simple queries + * This can be verified by checking that cursor is not present, and old default limit applies + */ + @Test + public void noPaginationWhenFetchSizeZero() throws IOException { + String selectQuery = StringUtils.format("SELECT firstname, state FROM %s", TEST_INDEX_ACCOUNT); + JSONObject response = new JSONObject(executeFetchQuery(selectQuery, 0, JDBC)); + assertFalse(response.has(CURSOR)); + assertThat(response.getJSONArray(DATAROWS).length(), equalTo(200)); + } + + /** + * The index has 1000 records, with fetch size of 50 we should get 20 pages with no cursor on last page + */ + @Test + public void validNumberOfPages() throws IOException { + String selectQuery = StringUtils.format("SELECT firstname, state FROM %s", TEST_INDEX_ACCOUNT); + JSONObject response = new JSONObject(executeFetchQuery(selectQuery, 50, JDBC)); + String cursor = response.getString(CURSOR); + int pageCount = 1; + + while (!cursor.isEmpty()) { //this condition also checks that there is no cursor on last page + response = executeCursorQuery(cursor); + cursor = response.optString(CURSOR); + pageCount++; + } + + assertThat(pageCount, equalTo(20)); + + // using random value here, with fetch size of 28 we should get 36 pages (ceil of 1000/28) + response = new JSONObject(executeFetchQuery(selectQuery, 28, JDBC)); + cursor = response.getString(CURSOR); + System.out.println(response); + pageCount = 1; + + while (!cursor.isEmpty()) { + response = executeCursorQuery(cursor); + cursor = response.optString(CURSOR); + pageCount++; + } + assertThat(pageCount, equalTo(36)); + } + + + @Test + public void validTotalResultWithAndWithoutPagination() throws IOException { + // simple query - accounts index has 1000 docs, using higher limit to get all docs + String selectQuery = StringUtils.format("SELECT firstname, state FROM %s ", TEST_INDEX_ACCOUNT); + verifyWithAndWithoutPaginationResponse(selectQuery + " LIMIT 2000", selectQuery, 80); + } + + @Test + public void validTotalResultWithAndWithoutPaginationWhereClause() throws IOException { + String selectQuery = StringUtils.format( + "SELECT firstname, state FROM %s WHERE balance < 25000 AND age > 32", TEST_INDEX_ACCOUNT + ); + verifyWithAndWithoutPaginationResponse(selectQuery + " LIMIT 2000", selectQuery, 17); + } + + @Test + public void validTotalResultWithAndWithoutPaginationOrderBy() throws IOException { + String selectQuery = StringUtils.format( + "SELECT firstname, state FROM %s ORDER BY balance DESC ", TEST_INDEX_ACCOUNT + ); + verifyWithAndWithoutPaginationResponse(selectQuery + " LIMIT 2000", selectQuery, 26); + } + + @Test + public void validTotalResultWithAndWithoutPaginationWhereAndOrderBy() throws IOException { + String selectQuery = StringUtils.format( + "SELECT firstname, state FROM %s WHERE balance < 25000 ORDER BY balance ASC ", + TEST_INDEX_ACCOUNT + ); + verifyWithAndWithoutPaginationResponse(selectQuery + " LIMIT 2000", selectQuery, 80); + + } + + @Test + public void validTotalResultWithAndWithoutPaginationNested() throws IOException { + loadIndex(Index.NESTED_SIMPLE); + String selectQuery = StringUtils.format( + "SELECT name, a.city, a.state FROM %s m , m.address as a ", TEST_INDEX_NESTED_SIMPLE + ); + verifyWithAndWithoutPaginationResponse(selectQuery + " LIMIT 2000", selectQuery, 1); + } + + @Test + public void noCursorWhenResultsLessThanFetchSize() throws IOException { + // fetch_size is 100, but actual number of rows returned from ElasticSearch is 97 + // a scroll context will be opened but will be closed after first page as all records are fetched + String selectQuery = StringUtils.format( + "SELECT * FROM %s WHERE balance < 25000 AND age > 36 LIMIT 2000", TEST_INDEX_ACCOUNT + ); + JSONObject response = new JSONObject(executeFetchQuery(selectQuery, 100, JDBC)); + assertFalse(response.has(CURSOR)); + } + + @Test + public void testCursorWithPreparedStatement() throws IOException { + JSONObject response = executeJDBCRequest(String.format("{" + + " \"fetch_size\": 200," + + " \"query\": \" SELECT age, state FROM %s WHERE age > ? OR state IN (?, ?)\"," + + " \"parameters\": [" + + " {" + + " \"type\": \"integer\"," + + " \"value\": 25" + + " }," + + " {" + + " \"type\": \"string\"," + + " \"value\": \"WA\"" + + " }," + + " {" + + " \"type\": \"string\"," + + " \"value\": \"UT\"" + + " }" + + " ]" + + "}", TestsConstants.TEST_INDEX_ACCOUNT)); + + assertTrue(response.has(CURSOR)); + } + + @Test + public void testRegressionOnDateFormatChange() throws IOException { + loadIndex(Index.DATETIME); + /** + * With pagination, the field should be date formatted to MySQL format as in + * @see PR #367 actualDateList = new ArrayList<>(); + String selectQuery = + StringUtils.format("SELECT login_time FROM %s LIMIT 500", TEST_INDEX_DATE_TIME); + JSONObject response = new JSONObject(executeFetchQuery(selectQuery, 1, JDBC)); + String cursor = response.getString(CURSOR); + actualDateList.add(response.getJSONArray(DATAROWS).getJSONArray(0).getString(0)); + + while (!cursor.isEmpty()) { + response = executeCursorQuery(cursor); + cursor = response.optString(CURSOR); + actualDateList.add(response.getJSONArray(DATAROWS).getJSONArray(0).getString(0)); + } + + List expectedDateList = Arrays.asList( + "2015-01-01 00:00:00.000", + "2015-01-01 12:10:30.000", + "1585882955", // by existing design, this is not formatted in MySQL standard format + "2020-04-08 06:10:30.000"); + + assertThat(actualDateList, equalTo(expectedDateList)); + } + + + @Test + public void defaultBehaviorWhenCursorSettingIsDisabled() throws IOException { + updateClusterSettings(new ClusterSetting(PERSISTENT, "opendistro.sql.cursor.enabled", "false")); + String query = StringUtils.format("SELECT firstname, email, state FROM %s", TEST_INDEX_ACCOUNT); + JSONObject response = new JSONObject(executeFetchQuery(query, 100, JDBC)); + assertFalse(response.has(CURSOR)); + + updateClusterSettings(new ClusterSetting(PERSISTENT, "opendistro.sql.cursor.enabled", "true")); + query = StringUtils.format("SELECT firstname, email, state FROM %s", TEST_INDEX_ACCOUNT); + response = new JSONObject(executeFetchQuery(query, 100, JDBC)); + assertTrue(response.has(CURSOR)); + + wipeAllClusterSettings(); + } + + + @Test + public void testCursorSettings() throws IOException { + // reverting enableCursorClusterSetting() in init() method before checking defaults + updateClusterSettings(new ClusterSetting(PERSISTENT, "opendistro.sql.cursor.enabled", null)); + + // Assert default cursor settings + JSONObject clusterSettings = getAllClusterSettings(); + assertThat(clusterSettings.query("/defaults/opendistro.sql.cursor.enabled"), equalTo("false")); + assertThat(clusterSettings.query("/defaults/opendistro.sql.cursor.fetch_size"), + equalTo("1000")); + assertThat(clusterSettings.query("/defaults/opendistro.sql.cursor.keep_alive"), equalTo("1m")); + + updateClusterSettings(new ClusterSetting(PERSISTENT, "opendistro.sql.cursor.enabled", "true")); + updateClusterSettings(new ClusterSetting(TRANSIENT, "opendistro.sql.cursor.fetch_size", "400")); + updateClusterSettings( + new ClusterSetting(PERSISTENT, "opendistro.sql.cursor.keep_alive", "200s")); + + clusterSettings = getAllClusterSettings(); + assertThat(clusterSettings.query("/persistent/opendistro.sql.cursor.enabled"), equalTo("true")); + assertThat(clusterSettings.query("/transient/opendistro.sql.cursor.fetch_size"), + equalTo("400")); + assertThat(clusterSettings.query("/persistent/opendistro.sql.cursor.keep_alive"), + equalTo("200s")); + + wipeAllClusterSettings(); + } + + + @Test + public void testDefaultFetchSizeFromClusterSettings() throws IOException { + // the default fetch size is 1000 + // using non-nested query here as page will have more rows on flattening + String query = StringUtils.format("SELECT firstname, email, state FROM %s", TEST_INDEX_ACCOUNT); + JSONObject response = new JSONObject(executeFetchLessQuery(query, JDBC)); + JSONArray datawRows = response.optJSONArray(DATAROWS); + assertThat(datawRows.length(), equalTo(1000)); + + updateClusterSettings(new ClusterSetting(TRANSIENT, "opendistro.sql.cursor.fetch_size", "786")); + response = new JSONObject(executeFetchLessQuery(query, JDBC)); + datawRows = response.optJSONArray(DATAROWS); + assertThat(datawRows.length(), equalTo(786)); + assertTrue(response.has(CURSOR)); + + wipeAllClusterSettings(); + } + + @Test + public void testCursorCloseAPI() throws IOException { + // multiple invocation of closing cursor should return success + // fetch page using old cursor should throw error + String selectQuery = StringUtils.format( + "SELECT firstname, state FROM %s WHERE balance > 100 and age < 40", TEST_INDEX_ACCOUNT); + JSONObject result = new JSONObject(executeFetchQuery(selectQuery, 50, JDBC)); + String cursor = result.getString(CURSOR); + + // Retrieving next 10 pages out of remaining 19 pages + for (int i = 0; i < 10; i++) { + result = executeCursorQuery(cursor); + cursor = result.optString(CURSOR); + } + //Closing the cursor + JSONObject closeResp = executeCursorCloseQuery(cursor); + assertThat(closeResp.getBoolean("succeeded"), equalTo(true)); + + //Closing the cursor multiple times is idempotent + for (int i = 0; i < 5; i++) { + closeResp = executeCursorCloseQuery(cursor); + assertThat(closeResp.getBoolean("succeeded"), equalTo(true)); + } + + // using the cursor after its cleared, will throw exception + Response response = null; + try { + JSONObject queryResult = executeCursorQuery(cursor); + } catch (ResponseException ex) { + response = ex.getResponse(); + } + + JSONObject resp = new JSONObject(TestUtils.getResponseBody(response)); + assertThat(resp.getInt("status"), equalTo(404)); + assertThat(resp.query("/error/reason"), equalTo("all shards failed")); + assertThat(resp.query("/error/caused_by/reason").toString(), + containsString("No search context found")); + assertThat(resp.query("/error/type"), equalTo("search_phase_execution_exception")); + } + + + @Test + public void invalidCursorIdNotDecodable() throws IOException { + // could be either not decode-able + String randomCursor = + "d:eyJzY2hlbWEiOlt7Im5hbWUiOiJmaXJzdG5hbWUiLCJ0eXBlIjoidGV4dCJ9LHsibmFtZSI6InN0Y"; + + Response response = null; + try { + JSONObject resp = executeCursorQuery(randomCursor); + } catch (ResponseException ex) { + response = ex.getResponse(); + } + + JSONObject resp = new JSONObject(TestUtils.getResponseBody(response)); + assertThat(resp.getInt("status"), equalTo(400)); + assertThat(resp.query("/error/type"), equalTo("illegal_argument_exception")); + } + + /** + * The index has 1000 records, with fetch size of 50 and LIMIT in place + * we should get Math.ceil(limit/fetchSize) pages and LIMIT number of rows. + * Basically it should not retrieve all records in presence of a smaller LIMIT value. + */ + @Test + public void respectLimitPassedInSelectClause() throws IOException { + int limit = 234; + String selectQuery = + StringUtils.format("SELECT age, balance FROM %s LIMIT %s", TEST_INDEX_ACCOUNT, limit); + JSONObject response = new JSONObject(executeFetchQuery(selectQuery, 50, JDBC)); + String cursor = response.getString(CURSOR); + int actualDataRowCount = response.getJSONArray(DATAROWS).length(); + int pageCount = 1; + + while (!cursor.isEmpty()) { + response = executeCursorQuery(cursor); + cursor = response.optString(CURSOR); + actualDataRowCount += response.getJSONArray(DATAROWS).length(); + pageCount++; + } + + assertThat(pageCount, equalTo(5)); + assertThat(actualDataRowCount, equalTo(limit)); + } + + + @Test + public void noPaginationWithNonJDBCFormat() throws IOException { + // checking for CSV, RAW format + String query = + StringUtils.format("SELECT firstname, email, state FROM %s LIMIT 2000", TEST_INDEX_ACCOUNT); + String csvResult = executeFetchQuery(query, 100, "csv"); + String[] rows = csvResult.split(NEW_LINE); + // all the 1001 records (+1 for header) are retrieved instead of fetch_size number of records + assertThat(rows.length, equalTo(1001)); + + String rawResult = executeFetchQuery(query, 100, "raw"); + rows = rawResult.split(NEW_LINE); + // all the 1000 records (NO headers) are retrieved instead of fetch_size number of records + assertThat(rows.length, equalTo(1000)); + } + + + public void verifyWithAndWithoutPaginationResponse(String sqlQuery, String cursorQuery, + int fetch_size) throws IOException { + // we are only checking here for schema and datarows + JSONObject withoutCursorResponse = new JSONObject(executeFetchQuery(sqlQuery, 0, JDBC)); + + JSONObject withCursorResponse = new JSONObject("{\"schema\":[],\"datarows\":[]}"); + JSONArray schema = withCursorResponse.getJSONArray(SCHEMA); + JSONArray dataRows = withCursorResponse.getJSONArray(DATAROWS); + + JSONObject response = new JSONObject(executeFetchQuery(cursorQuery, fetch_size, JDBC)); + response.optJSONArray(SCHEMA).forEach(schema::put); + response.optJSONArray(DATAROWS).forEach(dataRows::put); + + String cursor = response.getString(CURSOR); + while (!cursor.isEmpty()) { + response = executeCursorQuery(cursor); + response.optJSONArray(DATAROWS).forEach(dataRows::put); + cursor = response.optString(CURSOR); + } + + verifySchema(withoutCursorResponse.optJSONArray(SCHEMA), + withCursorResponse.optJSONArray(SCHEMA)); + verifyDataRows(withoutCursorResponse.optJSONArray(DATAROWS), + withCursorResponse.optJSONArray(DATAROWS)); + } + + public void verifySchema(JSONArray schemaOne, JSONArray schemaTwo) { + assertTrue(schemaOne.similar(schemaTwo)); + } + + public void verifyDataRows(JSONArray dataRowsOne, JSONArray dataRowsTwo) { + assertTrue(dataRowsOne.similar(dataRowsTwo)); + } + + private void enableCursorClusterSetting() throws IOException { + updateClusterSettings( + new ClusterSetting("persistent", "opendistro.sql.cursor.enabled", "true")); + } + + public String executeFetchAsStringQuery(String query, String fetchSize, String requestType) + throws IOException { + String endpoint = "/_opendistro/_sql?format=" + requestType; + String requestBody = makeRequest(query, fetchSize); + + Request sqlRequest = new Request("POST", endpoint); + sqlRequest.setJsonEntity(requestBody); + + Response response = client().performRequest(sqlRequest); + String responseString = getResponseBody(response, true); + return responseString; + } + + private String makeRequest(String query, String fetch_size) { + return String.format("{" + + " \"fetch_size\": \"%s\"," + + " \"query\": \"%s\"" + + "}", fetch_size, query); + } + + private JSONObject executeJDBCRequest(String requestBody) throws IOException { + Request sqlRequest = getSqlRequest(requestBody, false, JDBC); + return new JSONObject(executeRequest(sqlRequest)); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/CustomExternalTestCluster.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/CustomExternalTestCluster.java new file mode 100644 index 0000000000..5a03a4fff6 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/CustomExternalTestCluster.java @@ -0,0 +1,178 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static org.elasticsearch.test.ESTestCase.getTestTransportType; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.env.Environment; +import org.elasticsearch.http.HttpInfo; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.InternalTestCluster; +import org.elasticsearch.test.TestCluster; +import org.elasticsearch.transport.MockTransportClient; +import org.elasticsearch.transport.nio.MockNioTransportPlugin; + +import static org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest.Metric.HTTP; +import static org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest.Metric.SETTINGS; + +public class CustomExternalTestCluster extends TestCluster { + + private static final Logger logger = LogManager.getLogger(CustomExternalTestCluster.class); + + private static final AtomicInteger counter = new AtomicInteger(); + public static final String EXTERNAL_CLUSTER_PREFIX = "external_"; + + private final MockTransportClient client; + + private final InetSocketAddress[] httpAddresses; + + private final String clusterName; + + private final int numDataNodes; + private final int numMasterAndDataNodes; + + public CustomExternalTestCluster(Path tempDir, Settings additionalSettings, + Collection> pluginClasses, + TransportAddress... transportAddresses) { + super(0); + Settings.Builder clientSettingsBuilder = Settings.builder() + .put(additionalSettings) + .put("node.name", InternalTestCluster.TRANSPORT_CLIENT_PREFIX + EXTERNAL_CLUSTER_PREFIX + + counter.getAndIncrement()) + .put("client.transport.ignore_cluster_name", true) + .put(Environment.PATH_HOME_SETTING.getKey(), tempDir); + boolean addMockTcpTransport = additionalSettings.get(NetworkModule.TRANSPORT_TYPE_KEY) == null; + + if (addMockTcpTransport) { + String transport = getTestTransportType(); + clientSettingsBuilder.put(NetworkModule.TRANSPORT_TYPE_KEY, transport); + if (pluginClasses.contains(MockNioTransportPlugin.class) == false) { + pluginClasses = new ArrayList<>(pluginClasses); + if (transport.equals(MockNioTransportPlugin.MOCK_NIO_TRANSPORT_NAME)) { + pluginClasses.add(MockNioTransportPlugin.class); + } + } + } + Settings clientSettings = clientSettingsBuilder.build(); + MockTransportClient client = new MockTransportClient(clientSettings, pluginClasses); + try { + client.addTransportAddresses(transportAddresses); + NodesInfoResponse nodeInfos = + client.admin().cluster().prepareNodesInfo().clear() + .addMetrics(SETTINGS.metricName(), HTTP.metricName()) + .get(); + httpAddresses = new InetSocketAddress[nodeInfos.getNodes().size()]; + this.clusterName = nodeInfos.getClusterName().value(); + int dataNodes = 0; + int masterAndDataNodes = 0; + for (int i = 0; i < nodeInfos.getNodes().size(); i++) { + NodeInfo nodeInfo = nodeInfos.getNodes().get(i); + httpAddresses[i] = nodeInfo.getInfo(HttpInfo.class).address().publishAddress().address(); + if (DiscoveryNode.isDataNode(nodeInfo.getSettings())) { + dataNodes++; + masterAndDataNodes++; + } else if (DiscoveryNode.isMasterNode(nodeInfo.getSettings())) { + masterAndDataNodes++; + } + } + this.numDataNodes = dataNodes; + this.numMasterAndDataNodes = masterAndDataNodes; + this.client = client; + + logger.info("Setup ExternalTestCluster [{}] made of [{}] nodes", + nodeInfos.getClusterName().value(), size()); + } catch (Exception e) { + client.close(); + throw e; + } + } + + @Override + public void afterTest() { + + } + + @Override + public Client client() { + return client; + } + + @Override + public int size() { + return httpAddresses.length; + } + + @Override + public int numDataNodes() { + return numDataNodes; + } + + @Override + public int numDataAndMasterNodes() { + return numMasterAndDataNodes; + } + + @Override + public InetSocketAddress[] httpAddresses() { + return httpAddresses; + } + + @Override + public void close() throws IOException { + client.close(); + } + + /** + * This custom ExternalCluster class has ensureEstimatedStats() emptied out to prevent the transport client error + * made from making a request using a closed client after the tests are complete. + */ + @Override + public void ensureEstimatedStats() { + } + + @Override + public Iterable getClients() { + return Collections.singleton(client); + } + + @Override + public NamedWriteableRegistry getNamedWriteableRegistry() { + return client.getNamedWriteableRegistry(); + } + + @Override + public String getClusterName() { + return clusterName; + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/DateFormatIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/DateFormatIT.java new file mode 100644 index 0000000000..f4f695dbb5 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/DateFormatIT.java @@ -0,0 +1,245 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.is; + +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.google.common.collect.Ordering; +import java.io.IOException; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Test; + +public class DateFormatIT extends SQLIntegTestCase { + + private static final String SELECT_FROM = + "SELECT insert_time " + + "FROM " + TestsConstants.TEST_INDEX_ONLINE + " "; + + @Override + protected void init() throws Exception { + loadIndex(Index.ONLINE); + } + + /** + * All of the following tests use UTC as their date_format timezone as this is the same timezone of the data + * being queried. This is to prevent discrepancies in the Elasticsearch query and the actual field data that is + * being checked for the integration tests. + *

+ * Large LIMIT values were given for some of these queries since the default result size of the query is 200 and + * this ends up excluding some of the expected values causing the assertion to fail. LIMIT overrides this. + */ + + @Test + public void equalTo() throws SqlParseException { + assertThat( + dateQuery( + SELECT_FROM + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') = '2014-08-17'"), + contains("2014-08-17") + ); + } + + @Test + public void lessThan() throws SqlParseException { + assertThat( + dateQuery( + SELECT_FROM + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') < '2014-08-18'"), + contains("2014-08-17") + ); + } + + @Test + public void lessThanOrEqualTo() throws SqlParseException { + assertThat( + dateQuery( + SELECT_FROM + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') <= '2014-08-18' " + + "ORDER BY insert_time " + + "LIMIT 1000"), + contains("2014-08-17", "2014-08-18") + ); + } + + @Test + public void greaterThan() throws SqlParseException { + assertThat( + dateQuery( + SELECT_FROM + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') > '2014-08-23'"), + contains("2014-08-24") + ); + } + + @Test + public void greaterThanOrEqualTo() throws SqlParseException { + assertThat( + dateQuery( + SELECT_FROM + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') >= '2014-08-23' " + + "ORDER BY insert_time " + + "LIMIT 2000"), + contains("2014-08-23", "2014-08-24") + ); + } + + @Test + public void and() throws SqlParseException { + assertThat( + dateQuery(SELECT_FROM + + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') >= '2014-08-21' " + + "AND date_format(insert_time, 'yyyy-MM-dd', 'UTC') <= '2014-08-23' " + + "ORDER BY insert_time " + + "LIMIT 3000"), + contains("2014-08-21", "2014-08-22", "2014-08-23") + ); + } + + @Test + public void or() throws SqlParseException { + assertThat( + dateQuery(SELECT_FROM + + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') < '2014-08-18' " + + "OR date_format(insert_time, 'yyyy-MM-dd', 'UTC') > '2014-08-23' " + + "ORDER BY insert_time " + + "LIMIT 1000"), + contains("2014-08-17", "2014-08-24") + ); + } + + + @Test + public void sortByDateFormat() throws IOException { + // Sort by expression in descending order, but sort inside in ascending order, so we increase our confidence + // that successful test isn't just random chance. + + JSONArray hits = + getHits(executeQuery("SELECT all_client, insert_time " + + " FROM " + TestsConstants.TEST_INDEX_ONLINE + + " ORDER BY date_format(insert_time, 'dd-MM-YYYY', 'UTC') DESC, insert_time " + + " LIMIT 10")); + + assertThat(new DateTime(getSource(hits.getJSONObject(0)).get("insert_time"), DateTimeZone.UTC), + is(new DateTime("2014-08-24T00:00:41.221Z", DateTimeZone.UTC))); + } + + @Test + public void sortByAliasedDateFormat() throws IOException { + JSONArray hits = + getHits(executeQuery( + "SELECT all_client, insert_time, date_format(insert_time, 'dd-MM-YYYY', 'UTC') date" + + " FROM " + TestsConstants.TEST_INDEX_ONLINE + + " ORDER BY date DESC, insert_time " + + " LIMIT 10")); + + assertThat(new DateTime(getSource(hits.getJSONObject(0)).get("insert_time"), DateTimeZone.UTC), + is(new DateTime("2014-08-24T00:00:41.221Z", DateTimeZone.UTC))); + } + + @Test + public void groupByAndSort() throws IOException { + JSONObject aggregations = executeQuery( + "SELECT date_format(insert_time, 'dd-MM-YYYY') " + + "FROM elasticsearch-sql_test_index_online " + + "GROUP BY date_format(insert_time, 'dd-MM-YYYY') " + + "ORDER BY date_format(insert_time, 'dd-MM-YYYY') DESC") + .getJSONObject("aggregations"); + + checkAggregations(aggregations, "date_format", Ordering.natural().reverse()); + } + + @Test + public void groupByAndSortAliasedReversed() throws IOException { + JSONObject aggregations = executeQuery( + "SELECT date_format(insert_time, 'dd-MM-YYYY') date " + + "FROM elasticsearch-sql_test_index_online " + + "GROUP BY date " + + "ORDER BY date DESC") + .getJSONObject("aggregations"); + + checkAggregations(aggregations, "date", Ordering.natural().reverse()); + } + + @Test + public void groupByAndSortAliased() throws IOException { + JSONObject aggregations = executeQuery( + "SELECT date_format(insert_time, 'dd-MM-YYYY') date " + + "FROM elasticsearch-sql_test_index_online " + + "GROUP BY date " + + "ORDER BY date ") + .getJSONObject("aggregations"); + + checkAggregations(aggregations, "date", Ordering.natural()); + } + + private void checkAggregations(JSONObject aggregations, String key, + Ordering ordering) { + String date = getScriptAggregationKey(aggregations, key); + JSONArray buckets = aggregations.getJSONObject(date).getJSONArray("buckets"); + + assertThat(buckets.length(), is(8)); + + List aggregationSortKeys = IntStream.range(0, 8) + .mapToObj(index -> buckets.getJSONObject(index).getString("key")) + .collect(Collectors.toList()); + + assertTrue("The query result must be sorted by date in descending order", + ordering.isOrdered(aggregationSortKeys)); + } + + private Set dateQuery(String sql) throws SqlParseException { + try { + JSONObject response = executeQuery(sql); + return getResult(response, "insert_time"); + } catch (IOException e) { + throw new SqlParseException(String.format("Unable to process query '%s'", sql)); + } + } + + private Set getResult(JSONObject response, String fieldName) { + DateTimeFormatter formatter = DateTimeFormat.forPattern(TestsConstants.SIMPLE_DATE_FORMAT); + + JSONArray hits = getHits(response); + Set result = new TreeSet<>(); // Using TreeSet so order is maintained + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + JSONObject source = getSource(hit); + DateTime date = new DateTime(source.get(fieldName), DateTimeZone.UTC); + String formattedDate = formatter.print(date); + + result.add(formattedDate); + } + + return result; + } + + public static String getScriptAggregationKey(JSONObject aggregation, String prefix) { + return aggregation.keySet() + .stream() + .filter(x -> x.startsWith(prefix)) + .findFirst() + .orElseThrow(() -> new RuntimeException( + "Can't find key" + prefix + " in aggregation " + aggregation)); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/DateFunctionsIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/DateFunctionsIT.java new file mode 100644 index 0000000000..b3a281e5d4 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/DateFunctionsIT.java @@ -0,0 +1,324 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.matchesPattern; + +import java.io.IOException; +import java.time.Month; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.search.SearchHit; +import org.joda.time.DateTime; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.json.JSONObject; +import org.junit.Test; + +public class DateFunctionsIT extends SQLIntegTestCase { + + private static final String FROM = "FROM " + TestsConstants.TEST_INDEX_ONLINE; + + /** + * Some of the first few SQL functions are tested in both SELECT and WHERE cases for flexibility and the remainder + * are merely tested in SELECT for simplicity. + *

+ * There is a limitation in all date SQL functions in that they expect a date field as input. In the future this + * can be expanded on by supporting CAST and casting dates given as Strings to TIMESTAMP (SQL's date type). + */ + + @Override + protected void init() throws Exception { + loadIndex(Index.ONLINE); + } + + @Test + public void year() throws IOException { + SearchHit[] hits = query( + "SELECT YEAR(insert_time) as year" + ); + for (SearchHit hit : hits) { + int year = (int) getField(hit, "year"); + DateTime insertTime = getDateFromSource(hit, "insert_time"); + assertThat(year, equalTo(insertTime.year().get())); + } + } + + @Test + public void monthOfYear() throws IOException { + SearchHit[] hits = query( + "SELECT MONTH_OF_YEAR(insert_time) as month_of_year" + ); + for (SearchHit hit : hits) { + int monthOfYear = (int) getField(hit, "month_of_year"); + DateTime insertTime = getDateFromSource(hit, "insert_time"); + assertThat(monthOfYear, equalTo(insertTime.monthOfYear().get())); + } + } + + @Test + public void weekOfYearInSelect() throws IOException { + SearchHit[] hits = query( + "SELECT WEEK_OF_YEAR(insert_time) as week_of_year" + ); + for (SearchHit hit : hits) { + int weekOfYear = (int) getField(hit, "week_of_year"); + DateTime insertTime = getDateFromSource(hit, "insert_time"); + assertThat(weekOfYear, equalTo(insertTime.weekOfWeekyear().get())); + } + } + + @Test + public void weekOfYearInWhere() throws IOException { + SearchHit[] hits = query( + "SELECT insert_time", + "WHERE DATE_FORMAT(insert_time, 'YYYY-MM-dd') < '2014-08-19' AND " + + "WEEK_OF_YEAR(insert_time) > 33", + "LIMIT 2000" + ); + for (SearchHit hit : hits) { + DateTime insertTime = getDateFromSource(hit, "insert_time"); + assertThat(insertTime.weekOfWeekyear().get(), greaterThan(33)); + } + } + + @Test + public void dayOfYearInSelect() throws IOException { + SearchHit[] hits = query( + "SELECT DAY_OF_YEAR(insert_time) as day_of_year", "LIMIT 2000" + ); + for (SearchHit hit : hits) { + int dayOfYear = (int) getField(hit, "day_of_year"); + DateTime insertTime = getDateFromSource(hit, "insert_time"); + assertThat(dayOfYear, equalTo(insertTime.dayOfYear().get())); + } + } + + @Test + public void dayOfYearInWhere() throws IOException { + SearchHit[] hits = query( + "SELECT insert_time", "WHERE DAY_OF_YEAR(insert_time) < 233", "LIMIT 10000" + ); + for (SearchHit hit : hits) { + DateTime insertTime = getDateFromSource(hit, "insert_time"); + assertThat(insertTime.dayOfYear().get(), lessThan(233)); + } + } + + @Test + public void dayOfMonthInSelect() throws IOException { + SearchHit[] hits = query( + "SELECT DAY_OF_MONTH(insert_time) as day_of_month", "LIMIT 2000" + ); + for (SearchHit hit : hits) { + int dayOfMonth = (int) getField(hit, "day_of_month"); + DateTime insertTime = getDateFromSource(hit, "insert_time"); + assertThat(dayOfMonth, equalTo(insertTime.dayOfMonth().get())); + } + } + + @Test + public void dayOfMonthInWhere() throws IOException { + SearchHit[] hits = query( + "SELECT insert_time", "WHERE DAY_OF_MONTH(insert_time) < 21", "LIMIT 10000" + ); + for (SearchHit hit : hits) { + DateTime insertTime = getDateFromSource(hit, "insert_time"); + assertThat(insertTime.dayOfMonth().get(), lessThan(21)); + } + } + + @Test + public void dayOfWeek() throws IOException { + SearchHit[] hits = query( + "SELECT DAY_OF_WEEK(insert_time) as day_of_week", "LIMIT 2000" + ); + for (SearchHit hit : hits) { + int dayOfWeek = (int) getField(hit, "day_of_week"); + DateTime insertTime = getDateFromSource(hit, "insert_time"); + assertThat(dayOfWeek, equalTo(insertTime.dayOfWeek().get())); + } + } + + @Test + public void hourOfDay() throws IOException { + SearchHit[] hits = query( + "SELECT HOUR_OF_DAY(insert_time) as hour_of_day", "LIMIT 1000" + ); + for (SearchHit hit : hits) { + int hourOfDay = (int) getField(hit, "hour_of_day"); + DateTime insertTime = getDateFromSource(hit, "insert_time"); + assertThat(hourOfDay, equalTo(insertTime.hourOfDay().get())); + } + } + + @Test + public void minuteOfDay() throws IOException { + SearchHit[] hits = query( + "SELECT MINUTE_OF_DAY(insert_time) as minute_of_day", "LIMIT 500" + ); + for (SearchHit hit : hits) { + int minuteOfDay = (int) getField(hit, "minute_of_day"); + DateTime insertTime = getDateFromSource(hit, "insert_time"); + assertThat(minuteOfDay, equalTo(insertTime.minuteOfDay().get())); + } + } + + @Test + public void minuteOfHour() throws IOException { + SearchHit[] hits = query( + "SELECT MINUTE_OF_HOUR(insert_time) as minute_of_hour", "LIMIT 500" + ); + for (SearchHit hit : hits) { + int minuteOfHour = (int) getField(hit, "minute_of_hour"); + DateTime insertTime = getDateFromSource(hit, "insert_time"); + assertThat(minuteOfHour, equalTo(insertTime.minuteOfHour().get())); + } + } + + @Test + public void secondOfMinute() throws IOException { + SearchHit[] hits = query( + "SELECT SECOND_OF_MINUTE(insert_time) as second_of_minute", "LIMIT 500" + ); + for (SearchHit hit : hits) { + int secondOfMinute = (int) getField(hit, "second_of_minute"); + DateTime insertTime = getDateFromSource(hit, "insert_time"); + assertThat(secondOfMinute, equalTo(insertTime.secondOfMinute().get())); + } + } + + @Test + public void month() throws IOException { + SearchHit[] hits = query( + "SELECT MONTH(insert_time) AS month", "LIMIT 500" + ); + for (SearchHit hit : hits) { + int month = (int) getField(hit, "month"); + DateTime dateTime = getDateFromSource(hit, "insert_time"); + assertThat(month, equalTo(dateTime.monthOfYear().get())); + } + } + + @Test + public void dayofmonth() throws IOException { + SearchHit[] hits = query( + "SELECT DAYOFMONTH(insert_time) AS dayofmonth", "LIMIT 500" + ); + for (SearchHit hit : hits) { + int dayofmonth = (int) getField(hit, "dayofmonth"); + DateTime dateTime = getDateFromSource(hit, "insert_time"); + assertThat(dayofmonth, equalTo(dateTime.dayOfMonth().get())); + } + } + + @Test + public void date() throws IOException { + SearchHit[] hits = query( + "SELECT DATE(insert_time) AS date", "LIMIT 500" + ); + for (SearchHit hit : hits) { + String date = (String) getField(hit, "date"); + DateTime dateTime = getDateFromSource(hit, "insert_time"); + assertThat(date, equalTo(dateTime.toString("yyyy-MM-dd"))); + } + } + + @Test + public void monthname() throws IOException { + SearchHit[] hits = query( + "SELECT MONTHNAME(insert_time) AS monthname", "LIMIT 500" + ); + for (SearchHit hit : hits) { + String monthname = (String) getField(hit, "monthname"); + DateTime dateTime = getDateFromSource(hit, "insert_time"); + assertThat(Month.valueOf(monthname), equalTo(Month.of(dateTime.getMonthOfYear()))); + } + } + + @Test + public void timestamp() throws IOException { + SearchHit[] hits = query( + "SELECT TIMESTAMP(insert_time) AS timestamp", "LIMIT 500" + ); + for (SearchHit hit : hits) { + String timastamp = (String) getField(hit, "timestamp"); + DateTime dateTime = getDateFromSource(hit, "insert_time"); + assertThat(timastamp, equalTo(dateTime.toString("yyyy-MM-dd HH:mm:ss"))); + } + } + + @Test + public void maketime() throws IOException { + SearchHit[] hits = query("SELECT MAKETIME(13, 1, 1) AS maketime"); + String maketime = (String) getField(hits[0], "maketime"); + assertThat(maketime, equalTo("13:01:01")); + } + + @Test + public void now() throws IOException { + SearchHit[] hits = query("SELECT NOW() AS now"); + String now = (String) getField(hits[0], "now"); + assertThat(now, matchesPattern("[0-9]{2}:[0-9]{2}:[0-9]{2}")); + } + + @Test + public void curdate() throws IOException { + SearchHit[] hits = query("SELECT CURDATE() AS curdate"); + String curdate = (String) getField(hits[0], "curdate"); + assertThat(curdate, matchesPattern("[0-9]{4}-[0-9]{2}-[0-9]{2}")); + } + + private SearchHit[] query(String select, String... statements) throws IOException { + return execute(select + " " + FROM + " " + String.join(" ", statements)); + } + + // TODO: I think this code is now re-used in multiple classes, would be good to move to the base class. + private SearchHit[] execute(String sqlRequest) throws IOException { + final JSONObject jsonObject = executeRequest(makeRequest(sqlRequest)); + + final XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser( + NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, + jsonObject.toString()); + return SearchResponse.fromXContent(parser).getHits().getHits(); + } + + private Object getField(SearchHit hit, String fieldName) { + return hit.field(fieldName).getValue(); + } + + private Object getFieldFromSource(SearchHit hit, String fieldName) { + return hit.getSourceAsMap().get(fieldName); + } + + private DateTime getDateTime(String date) { + DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + return formatter.parseDateTime(date); + } + + private DateTime getDateFromSource(SearchHit hit, String dateField) { + return getDateTime((String) getFieldFromSource(hit, dateField)); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/DeleteIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/DeleteIT.java new file mode 100644 index 0000000000..7d408aadce --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/DeleteIT.java @@ -0,0 +1,143 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static org.hamcrest.core.IsEqual.equalTo; + +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; +import java.io.IOException; +import org.json.JSONObject; +import org.junit.Test; + +public class DeleteIT extends SQLIntegTestCase { + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + loadIndex(Index.PHRASE); + } + + @Test + public void deleteAllTest() throws IOException, InterruptedException { + String selectQuery = StringUtils.format("SELECT * FROM %s", TestsConstants.TEST_INDEX_ACCOUNT); + JSONObject response = executeRequest(makeRequest(selectQuery)); + int totalHits = getTotalHits(response); + + String deleteQuery = StringUtils.format("DELETE FROM %s", TestsConstants.TEST_INDEX_ACCOUNT); + response = executeRequest(makeRequest(deleteQuery)); + assertThat(response.getInt("deleted"), equalTo(totalHits)); + + // The documents are not deleted immediately, causing the next search call to return all results. + // To prevent flakiness, the minimum value of 2000 msec works fine. + Thread.sleep(2000); + + response = executeRequest(makeRequest(selectQuery)); + assertThat(getTotalHits(response), equalTo(0)); + } + + @Test + public void deleteWithConditionTest() throws IOException, InterruptedException { + String selectQuery = StringUtils.format( + "SELECT * FROM %s WHERE match_phrase(phrase, 'quick fox here')", + TestsConstants.TEST_INDEX_PHRASE + ); + JSONObject response = executeRequest(makeRequest(selectQuery)); + int totalHits = getTotalHits(response); + + String deleteQuery = StringUtils.format( + "DELETE FROM %s WHERE match_phrase(phrase, 'quick fox here')", + TestsConstants.TEST_INDEX_PHRASE + ); + response = executeRequest(makeRequest(deleteQuery)); + assertThat(response.getInt("deleted"), equalTo(totalHits)); + // The documents are not deleted immediately, causing the next search call to return all results. + // To prevent flakiness, the minimum value of 2000 msec works fine. + Thread.sleep(2000); + + selectQuery = StringUtils.format("SELECT * FROM %s", TestsConstants.TEST_INDEX_PHRASE); + + response = executeRequest(makeRequest(selectQuery)); + assertThat(getTotalHits(response), equalTo(5)); + } + + @Test + public void deleteAllWithJdbcFormat() throws IOException, InterruptedException { + String selectQuery = StringUtils.format("SELECT * FROM %s", TestsConstants.TEST_INDEX_ACCOUNT); + JSONObject response = executeRequest(makeRequest(selectQuery)); + int totalHits = getTotalHits(response); + + String deleteQuery = StringUtils.format("DELETE FROM %s", TestsConstants.TEST_INDEX_ACCOUNT); + + response = new JSONObject(executeQuery(deleteQuery, "jdbc")); + System.out.println(response); + assertThat(response.query("/schema/0/name"), equalTo("deleted_rows")); + assertThat(response.query("/schema/0/type"), equalTo("long")); + assertThat(response.query("/datarows/0/0"), equalTo(totalHits)); + assertThat(response.query("/total"), equalTo(1)); + assertThat(response.query("/status"), equalTo(200)); + assertThat(response.query("/size"), equalTo(1)); + + // The documents are not deleted immediately, causing the next search call to return all results. + // To prevent flakiness, the minimum value of 2000 msec works fine. + Thread.sleep(2000); + + response = executeRequest(makeRequest(selectQuery)); + assertThat(getTotalHits(response), equalTo(0)); + + // Multiple invocation of delete query should return deleted == 0 + response = new JSONObject(executeQuery(deleteQuery, "jdbc")); + assertThat(response.query("/datarows/0/0"), equalTo(0)); + } + + @Test + public void deleteWithConditionTestJdbcFormat() throws IOException, InterruptedException { + String selectQuery = StringUtils.format( + "SELECT * FROM %s WHERE match_phrase(phrase, 'quick fox here')", + TestsConstants.TEST_INDEX_PHRASE + ); + + JSONObject response = executeRequest(makeRequest(selectQuery)); + int totalHits = getTotalHits(response); + + String deleteQuery = StringUtils.format( + "DELETE FROM %s WHERE match_phrase(phrase, 'quick fox here')", + TestsConstants.TEST_INDEX_PHRASE + ); + + response = new JSONObject(executeQuery(deleteQuery, "jdbc")); + System.out.println(response); + assertThat(response.query("/schema/0/name"), equalTo("deleted_rows")); + assertThat(response.query("/schema/0/type"), equalTo("long")); + assertThat(response.query("/datarows/0/0"), equalTo(totalHits)); + assertThat(response.query("/total"), equalTo(1)); + assertThat(response.query("/status"), equalTo(200)); + assertThat(response.query("/size"), equalTo(1)); + + // The documents are not deleted immediately, causing the next search call to return all results. + // To prevent flakiness, the minimum value of 2000 msec works fine. + Thread.sleep(2000); + + selectQuery = StringUtils.format("SELECT * FROM %s", TestsConstants.TEST_INDEX_PHRASE); + + response = executeRequest(makeRequest(selectQuery)); + assertThat(getTotalHits(response), equalTo(5)); + + // Multiple invocation of delete query should return deleted == 0 + response = new JSONObject(executeQuery(deleteQuery, "jdbc")); + assertThat(response.query("/datarows/0/0"), equalTo(0)); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/ExplainIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/ExplainIT.java new file mode 100644 index 0000000000..d1f0a134e8 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/ExplainIT.java @@ -0,0 +1,233 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_DOG; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_LOCATION; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_TYPE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_PEOPLE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_PHRASE; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +import com.google.common.io.Files; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + +public class ExplainIT extends SQLIntegTestCase { + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + loadIndex(Index.DOG); + loadIndex(Index.PEOPLE); + loadIndex(Index.PHRASE); + loadIndex(Index.LOCATION); + loadIndex(Index.NESTED); + } + + @Test + public void searchSanity() throws IOException { + + String expectedOutputFilePath = TestUtils.getResourceFilePath( + "src/test/resources/expectedOutput/search_explain.json"); + String expectedOutput = Files.toString(new File(expectedOutputFilePath), StandardCharsets.UTF_8) + .replaceAll("\r", ""); + + String result = + explainQuery(String.format("SELECT * FROM %s WHERE firstname LIKE 'A%%' AND age > 20 " + + "GROUP BY gender order by _score", TEST_INDEX_ACCOUNT)); + Assert + .assertThat(result.replaceAll("\\s+", ""), equalTo(expectedOutput.replaceAll("\\s+", ""))); + } + + // This test was ignored because group by case function is not supported + @Ignore + @Test + public void aggregationQuery() throws IOException { + + String expectedOutputFilePath = TestUtils.getResourceFilePath( + "src/test/resources/expectedOutput/aggregation_query_explain.json"); + String expectedOutput = Files.toString(new File(expectedOutputFilePath), StandardCharsets.UTF_8) + .replaceAll("\r", ""); + + String result = explainQuery( + String.format("SELECT address, CASE WHEN gender='0' then 'aaa' else 'bbb'end a2345," + + "count(age) FROM %s GROUP BY terms('field'='address','execution_hint'='global_ordinals'),a2345", + TEST_INDEX_ACCOUNT)); + Assert + .assertThat(result.replaceAll("\\s+", ""), equalTo(expectedOutput.replaceAll("\\s+", ""))); + } + + @Test + public void explainScriptValue() throws IOException { + + String expectedOutputFilePath = TestUtils.getResourceFilePath( + "src/test/resources/expectedOutput/script_value.json"); + String expectedOutput = Files.toString(new File(expectedOutputFilePath), StandardCharsets.UTF_8) + .replaceAll("\r", ""); + + String result = explainQuery(String.format("SELECT case when gender is null then 'aaa' " + + "else gender end test , account_number FROM %s", TEST_INDEX_ACCOUNT)); + Assert + .assertThat(result.replaceAll("\\s+", ""), equalTo(expectedOutput.replaceAll("\\s+", ""))); + } + + @Test + public void betweenScriptValue() throws IOException { + + String expectedOutputFilePath = TestUtils.getResourceFilePath( + "src/test/resources/expectedOutput/between_query.json"); + String expectedOutput = Files.toString(new File(expectedOutputFilePath), StandardCharsets.UTF_8) + .replaceAll("\r", ""); + + String result = + explainQuery(String.format("SELECT case when balance between 100 and 200 then 'aaa' " + + "else balance end test, account_number FROM %s", TEST_INDEX_ACCOUNT)); + Assert + .assertThat(result.replaceAll("\\s+", ""), equalTo(expectedOutput.replaceAll("\\s+", ""))); + } + + @Test + public void searchSanityFilter() throws IOException { + + String expectedOutputFilePath = TestUtils.getResourceFilePath( + "src/test/resources/expectedOutput/search_explain_filter.json"); + String expectedOutput = Files.toString(new File(expectedOutputFilePath), StandardCharsets.UTF_8) + .replaceAll("\r", ""); + + String result = explainQuery(String.format("SELECT * FROM %s WHERE firstname LIKE 'A%%' " + + "AND age > 20 GROUP BY gender", TEST_INDEX_ACCOUNT)); + Assert + .assertThat(result.replaceAll("\\s+", ""), equalTo(expectedOutput.replaceAll("\\s+", ""))); + } + + @Test + public void deleteSanity() throws IOException { + + String expectedOutputFilePath = TestUtils.getResourceFilePath( + "src/test/resources/expectedOutput/delete_explain.json"); + String expectedOutput = Files.toString(new File(expectedOutputFilePath), StandardCharsets.UTF_8) + .replaceAll("\r", ""); + ; + + String result = + explainQuery(String.format("DELETE FROM %s WHERE firstname LIKE 'A%%' AND age > 20", + TEST_INDEX_ACCOUNT)); + Assert + .assertThat(result.replaceAll("\\s+", ""), equalTo(expectedOutput.replaceAll("\\s+", ""))); + } + + @Test + public void spatialFilterExplainTest() throws IOException { + + String expectedOutputFilePath = TestUtils.getResourceFilePath( + "src/test/resources/expectedOutput/search_spatial_explain.json"); + String expectedOutput = Files.toString(new File(expectedOutputFilePath), StandardCharsets.UTF_8) + .replaceAll("\r", ""); + ; + + String result = explainQuery(String.format("SELECT * FROM %s WHERE GEO_INTERSECTS" + + "(place,'POLYGON ((102 2, 103 2, 103 3, 102 3, 102 2))')", TEST_INDEX_LOCATION)); + Assert + .assertThat(result.replaceAll("\\s+", ""), equalTo(expectedOutput.replaceAll("\\s+", ""))); + } + + @Test + public void orderByOnNestedFieldTest() throws Exception { + + String result = + explainQuery(String.format("SELECT * FROM %s ORDER BY NESTED('message.info','message')", + TEST_INDEX_NESTED_TYPE)); + Assert.assertThat(result.replaceAll("\\s+", ""), + equalTo("{\"from\":0,\"size\":200,\"sort\":[{\"message.info\":" + + "{\"order\":\"asc\",\"nested\":{\"path\":\"message\"}}}]}")); + } + + @Test + public void multiMatchQuery() throws IOException { + + String expectedOutputFilePath = TestUtils.getResourceFilePath( + "src/test/resources/expectedOutput/multi_match_query.json"); + String expectedOutput = Files.toString(new File(expectedOutputFilePath), StandardCharsets.UTF_8) + .replaceAll("\r", ""); + + String result = + explainQuery(String.format("SELECT * FROM %s WHERE multimatch('query'='this is a test'," + + "'fields'='subject^3,message','analyzer'='standard','type'='best_fields','boost'=1.0," + + "'slop'=0,'tie_breaker'=0.3,'operator'='and')", TEST_INDEX_ACCOUNT)); + Assert + .assertThat(result.replaceAll("\\s+", ""), equalTo(expectedOutput.replaceAll("\\s+", ""))); + } + + @Test + public void termsIncludeExcludeExplainTest() throws IOException { + + final String queryPrefix = "SELECT * FROM " + TEST_INDEX_PHRASE + " GROUP BY "; + final String expected1 = "\"include\":\".*sport.*\",\"exclude\":\"water_.*\""; + final String expected2 = "\"include\":[\"honda\",\"mazda\"],\"exclude\":[\"jensen\",\"rover\"]"; + final String expected3 = "\"include\":{\"partition\":0,\"num_partitions\":20}"; + + String result = + explainQuery(queryPrefix + " terms('field'='correspond_brand_name','size'='10'," + + "'alias'='correspond_brand_name','include'='\\\".*sport.*\\\"','exclude'='\\\"water_.*\\\"')"); + Assert.assertThat(result, containsString(expected1)); + + result = explainQuery(queryPrefix + "terms('field'='correspond_brand_name','size'='10'," + + "'alias'='correspond_brand_name','include'='[\\\"mazda\\\", \\\"honda\\\"]'," + + "'exclude'='[\\\"rover\\\", \\\"jensen\\\"]')"); + Assert.assertThat(result, containsString(expected2)); + + result = explainQuery(queryPrefix + "terms('field'='correspond_brand_name','size'='10'," + + "'alias'='correspond_brand_name','include'='{\\\"partition\\\":0,\\\"num_partitions\\\":20}')"); + Assert.assertThat(result, containsString(expected3)); + } + + @Test + public void explainNLJoin() throws IOException { + + String expectedOutputFilePath = TestUtils.getResourceFilePath( + "src/test/resources/expectedOutput/nested_loop_join_explain.json"); + String expectedOutput = Files.toString(new File(expectedOutputFilePath), StandardCharsets.UTF_8) + .replaceAll("\r", ""); + + String query = "SELECT /*! USE_NL*/ a.firstname ,a.lastname , a.gender ,d.dog_name FROM " + + TEST_INDEX_PEOPLE + "/people a JOIN " + TEST_INDEX_DOG + + "/dog d on d.holdersName = a.firstname" + + " WHERE (a.age > 10 OR a.balance > 2000) AND d.age > 1"; + String result = explainQuery(query); + + Assert + .assertThat(result.replaceAll("\\s+", ""), equalTo(expectedOutput.replaceAll("\\s+", ""))); + } + + public void testContentTypeOfExplainRequestShouldBeJson() throws IOException { + String query = makeRequest("SELECT firstname FROM elasticsearch-sql_test_index_account"); + Request request = getSqlRequest(query, true); + + Response response = client().performRequest(request); + + assertEquals("application/json; charset=UTF-8", response.getHeader("content-type")); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/GetEndpointQueryIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/GetEndpointQueryIT.java new file mode 100644 index 0000000000..5b7875da01 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/GetEndpointQueryIT.java @@ -0,0 +1,47 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; + +import java.io.IOException; +import org.elasticsearch.client.ResponseException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +/** + * Tests to cover requests with "?format=csv" parameter + */ +public class GetEndpointQueryIT extends SQLIntegTestCase { + + @Rule + public ExpectedException rule = ExpectedException.none(); + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + } + + @Test + public void getEndPointShouldBeInvalid() throws IOException { + rule.expect(ResponseException.class); + rule.expectMessage("Incorrect HTTP method"); + String query = "select name from " + TEST_INDEX_ACCOUNT; + executeQueryWithGetRequest(query); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/HashJoinIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/HashJoinIT.java new file mode 100644 index 0000000000..5fe8f0d358 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/HashJoinIT.java @@ -0,0 +1,184 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_GAME_OF_THRONES; +import static org.hamcrest.Matchers.equalTo; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Locale; +import java.util.Set; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Assert; +import org.junit.Test; + +/** + * Test new hash join algorithm by comparison with old implementation. + */ +public class HashJoinIT extends SQLIntegTestCase { + + /** + * Hint to use old join algorithm + */ + private static final String USE_OLD_JOIN_ALGORITHM = "/*! USE_NL*/"; + + /** + * Set limit to 100% to bypass circuit break check + */ + private static final String BYPASS_CIRCUIT_BREAK = "/*! JOIN_CIRCUIT_BREAK_LIMIT(100)*/"; + + /** + * Enable term filter optimization + */ + private static final String ENABLE_TERMS_FILTER = "/*! HASH_WITH_TERMS_FILTER*/"; + + /** + * Default page size is greater than block size + */ + private static final String PAGE_SIZE_GREATER_THAN_BLOCK_SIZE = + "/*! JOIN_ALGORITHM_BLOCK_SIZE(5)*/"; + + /** + * Page size is smaller than block size + */ + private static final String PAGE_SIZE_LESS_THAN_BLOCK_SIZE = + "/*! JOIN_ALGORITHM_BLOCK_SIZE(5)*/ /*! JOIN_SCROLL_PAGE_SIZE(2)*/"; + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + loadIndex(Index.GAME_OF_THRONES); + } + + @Test + public void innerJoin() throws IOException { + + testJoin("INNER JOIN"); + } + + @Test + public void leftJoin() throws IOException { + + testJoin("LEFT JOIN"); + } + + @Test + public void innerJoinWithObjectField() throws IOException { + testJoinWithObjectField("INNER JOIN", ""); + } + + @Test + public void leftJoinWithObjectField() throws IOException { + testJoinWithObjectField("LEFT JOIN", ""); + } + + @Test + public void innerJoinWithObjectFieldAndTermsFilter() throws IOException { + testJoinWithObjectField("INNER JOIN", ENABLE_TERMS_FILTER); + } + + @Test + public void leftJoinWithObjectFieldAndTermsFilter() throws IOException { + testJoinWithObjectField("LEFT JOIN", ENABLE_TERMS_FILTER); + } + + @Test + public void innerJoinWithObjectFieldAndPageSizeGreaterThanBlockSize() throws IOException { + testJoinWithObjectField("INNER JOIN", PAGE_SIZE_GREATER_THAN_BLOCK_SIZE); + } + + @Test + public void leftJoinWithObjectFieldAndPageSizeGreaterThanBlockSize() throws IOException { + testJoinWithObjectField("LEFT JOIN", PAGE_SIZE_GREATER_THAN_BLOCK_SIZE); + } + + @Test + public void innerJoinWithObjectFieldAndPageSizeLessThanBlockSize() throws IOException { + testJoinWithObjectField("INNER JOIN", PAGE_SIZE_LESS_THAN_BLOCK_SIZE); + } + + @Test + public void leftJoinWithObjectFieldAndPageSizeLessThanBlockSize() throws IOException { + testJoinWithObjectField("LEFT JOIN", PAGE_SIZE_LESS_THAN_BLOCK_SIZE); + } + + private void testJoin(final String join) throws IOException { + + final String queryPrefix = "SELECT"; + + // TODO: reduce the balance threshold to 10000 when the memory circuit breaker issue + // (https://github.com/opendistro-for-elasticsearch/sql/issues/73) is fixed. + final String querySuffixTemplate = + "a.firstname, a.lastname, b.city, b.state FROM %1$s a %2$s %1$s b " + + "ON b.age = a.age WHERE a.balance > 45000 AND b.age > 25 LIMIT 1000000"; + final String querySuffix = + String.format(Locale.ROOT, querySuffixTemplate, TEST_INDEX_ACCOUNT, join); + + final String oldQuery = String.join(" ", queryPrefix, USE_OLD_JOIN_ALGORITHM, querySuffix); + final String newQuery = String.join(" ", queryPrefix, BYPASS_CIRCUIT_BREAK, querySuffix); + + executeAndCompareOldAndNewJoins(oldQuery, newQuery); + } + + private void testJoinWithObjectField(final String join, final String hint) throws IOException { + + final String queryPrefix = "SELECT"; + + // TODO: reduce the balance threshold to 10000 when the memory circuit breaker issue + // (https://github.com/opendistro-for-elasticsearch/sql/issues/73) is fixed. + final String querySuffixTemplate = "c.name.firstname, c.name.lastname, f.hname, f.seat " + + "FROM %1$s c %2$s %1$s f ON f.gender.keyword = c.gender.keyword " + + "AND f.house.keyword = c.house.keyword " + + "WHERE c.gender = 'M' LIMIT 1000000"; + final String querySuffix = + String.format(Locale.ROOT, querySuffixTemplate, TEST_INDEX_GAME_OF_THRONES, join); + + final String oldQuery = String.join(" ", queryPrefix, USE_OLD_JOIN_ALGORITHM, querySuffix); + final String newQuery = String.join(" ", queryPrefix, hint, BYPASS_CIRCUIT_BREAK, querySuffix); + + executeAndCompareOldAndNewJoins(oldQuery, newQuery); + } + + private void executeAndCompareOldAndNewJoins(final String oldQuery, final String newQuery) + throws IOException { + + final JSONObject responseOld = executeQuery(oldQuery); + final JSONObject responseNew = executeQuery(newQuery); + + Assert.assertThat(getTotalHits(responseOld), equalTo(getTotalHits(responseNew))); + + final JSONArray hitsOld = getHits(responseOld); + final JSONArray hitsNew = getHits(responseNew); + + Assert.assertThat(hitsOld.length(), equalTo(hitsNew.length())); + + Set idsOld = new HashSet<>(); + + hitsOld.forEach(hitObj -> { + JSONObject hit = (JSONObject) hitObj; + idsOld.add(hit.getString("_id")); + }); + + hitsNew.forEach(hitObj -> { + JSONObject hit = (JSONObject) hitObj; + Assert.assertTrue(idsOld.contains(hit.getString("_id"))); + }); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/HavingIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/HavingIT.java new file mode 100644 index 0000000000..7635bafe3d --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/HavingIT.java @@ -0,0 +1,212 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static org.hamcrest.Matchers.arrayContaining; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.is; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.stream.Collectors; +import org.hamcrest.Matcher; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Test; + +public class HavingIT extends SQLIntegTestCase { + + private static final String SELECT_FROM_WHERE_GROUP_BY = + "SELECT state, COUNT(*) cnt " + + "FROM " + TestsConstants.TEST_INDEX_ACCOUNT + " " + + "WHERE age = 30 " + + "GROUP BY state "; + + private static final Set> states1 = rowSet(1, Arrays.asList( + "AK", "AR", "CT", "DE", "HI", "IA", "IL", "IN", "LA", "MA", "MD", "MN", + "MO", "MT", "NC", "ND", "NE", "NH", "NJ", "NV", "SD", "VT", "WV", "WY" + )); + private static final Set> states2 = + rowSet(2, Arrays.asList("AZ", "DC", "KS", "ME")); + private static final Set> states3 = + rowSet(3, Arrays.asList("AL", "ID", "KY", "OR", "TN")); + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + } + + @Test + public void equalsTo() throws IOException { + assertThat( + query(SELECT_FROM_WHERE_GROUP_BY + "HAVING cnt = 2"), + resultSet( + states2 + ) + ); + } + + @Test + public void lessThanOrEqual() throws IOException { + assertThat( + query(SELECT_FROM_WHERE_GROUP_BY + "HAVING cnt <= 2"), + resultSet( + states1, + states2 + ) + ); + } + + @Test + public void notEqualsTo() throws IOException { + assertThat( + query(SELECT_FROM_WHERE_GROUP_BY + "HAVING cnt <> 2"), + resultSet( + states1, + states3 + ) + ); + } + + @Test + public void between() throws IOException { + assertThat( + query(SELECT_FROM_WHERE_GROUP_BY + "HAVING cnt BETWEEN 1 AND 2"), + resultSet( + states1, + states2 + ) + ); + } + + @Test + public void notBetween() throws IOException { + assertThat( + query(SELECT_FROM_WHERE_GROUP_BY + "HAVING cnt NOT BETWEEN 1 AND 2"), + resultSet( + states3 + ) + ); + } + + @Test + public void in() throws IOException { + assertThat( + query(SELECT_FROM_WHERE_GROUP_BY + "HAVING cnt IN (2, 3)"), + resultSet( + states2, + states3 + ) + ); + } + + @Test + public void notIn() throws IOException { + assertThat( + query(SELECT_FROM_WHERE_GROUP_BY + "HAVING cnt NOT IN (2, 3)"), + resultSet( + states1 + ) + ); + } + + @Test + public void and() throws IOException { + assertThat( + query(SELECT_FROM_WHERE_GROUP_BY + "HAVING cnt >= 1 AND cnt < 3"), + resultSet( + states1, + states2 + ) + ); + } + + @Test + public void or() throws IOException { + assertThat( + query(SELECT_FROM_WHERE_GROUP_BY + "HAVING cnt = 1 OR cnt = 3"), + resultSet( + states1, + states3 + ) + ); + } + + @Test + public void not() throws IOException { + assertThat( + query(SELECT_FROM_WHERE_GROUP_BY + "HAVING NOT cnt >= 2"), + resultSet( + states1 + ) + ); + } + + @Test + public void notAndOr() throws IOException { + assertThat( + query(SELECT_FROM_WHERE_GROUP_BY + "HAVING NOT (cnt > 0 AND cnt <= 2)"), + resultSet( + states3 + ) + ); + } + + private Set query(String query) throws IOException { + JSONObject response = executeQuery(query); + return getResult(response, "state.keyword", "cnt"); + } + + private Set getResult(JSONObject response, String aggName, String aggFunc) { + + String bucketsPath = String.format(Locale.ROOT, "/aggregations/%s/buckets", aggName); + JSONArray buckets = (JSONArray) response.query(bucketsPath); + + Set result = new HashSet<>(); + for (int i = 0; i < buckets.length(); i++) { + JSONObject bucket = buckets.getJSONObject(i); + result.add(new Object[] { + bucket.get("key"), + ((JSONObject) bucket.get(aggFunc)).getLong("value") + }); + } + + return result; + } + + @SafeVarargs + private final Matcher> resultSet(Set>... rowSets) { + return containsInAnyOrder(Arrays.stream(rowSets) + .flatMap(Collection::stream) + .collect(Collectors.toList())); + } + + private static Set> rowSet(long count, List states) { + return states.stream() + .map(state -> row(state, count)) + .collect(Collectors.toSet()); + } + + private static Matcher row(String state, long count) { + return arrayContaining(is(state), is(count)); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/JSONRequestIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/JSONRequestIT.java new file mode 100644 index 0000000000..58529f0940 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/JSONRequestIT.java @@ -0,0 +1,203 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static org.hamcrest.CoreMatchers.anyOf; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.lessThan; + +import java.io.IOException; +import java.util.Map; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.json.JSONObject; +import org.junit.Test; + +public class JSONRequestIT extends SQLIntegTestCase { + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + loadIndex(Index.NESTED); + } + + @Test + public void search() throws IOException { + int ageToCompare = 25; + SearchHits response = query(String.format("{\"query\":\"" + + "SELECT * " + + "FROM %s " + + "WHERE age > %s " + + "LIMIT 1000\"}", TestsConstants.TEST_INDEX_ACCOUNT, ageToCompare)); + SearchHit[] hits = response.getHits(); + for (SearchHit hit : hits) { + int age = (int) hit.getSourceAsMap().get("age"); + assertThat(age, greaterThan(ageToCompare)); + } + } + + @Test + public void searchWithFilterAndNoWhere() throws IOException { + /* + * Human readable format of the request defined below: + * { + * "query": "SELECT * FROM accounts LIMIT 1000", + * "filter": { + * "range": { + * "age": { + * "gt": 25 + * } + * } + * } + * } + */ + int ageToCompare = 25; + SearchHits response = query(String.format("{\"query\":\"" + + "SELECT * " + + "FROM %s " + + "LIMIT 1000\",\"filter\":{\"range\":{\"age\":{\"gt\":%s}}}}", + TestsConstants.TEST_INDEX_ACCOUNT, ageToCompare)); + SearchHit[] hits = response.getHits(); + for (SearchHit hit : hits) { + int age = (int) hit.getSourceAsMap().get("age"); + assertThat(age, greaterThan(ageToCompare)); + } + } + + @Test + public void searchWithRangeFilter() throws IOException { + /* + * Human readable format of the request defined below: + * { + * "query": "SELECT * FROM accounts WHERE age > 25 LIMIT 1000", + * "filter": { + * "range": { + * "balance": { + * "lt": 35000 + * } + * } + * } + * } + */ + int ageToCompare = 25; + int balanceToCompare = 35000; + SearchHits response = query(String.format("{\"query\":\"" + + "SELECT * " + + "FROM %s " + + "WHERE age > %s " + + "LIMIT 1000\",\"filter\":{\"range\":{\"balance\":{\"lt\":%s}}}}", + TestsConstants.TEST_INDEX_ACCOUNT, ageToCompare, balanceToCompare)); + SearchHit[] hits = response.getHits(); + for (SearchHit hit : hits) { + int age = (int) hit.getSourceAsMap().get("age"); + int balance = (int) hit.getSourceAsMap().get("balance"); + assertThat(age, greaterThan(ageToCompare)); + assertThat(balance, lessThan(balanceToCompare)); + } + } + + @Test + /** + * Using TEST_INDEX_NESTED_TYPE here since term filter does not work properly on analyzed fields like text. + * The field 'someField' in TEST_INDEX_NESTED_TYPE is of type keyword. + */ + public void searchWithTermFilter() throws IOException { + /* + * Human readable format of the request defined below: + * { + * "query": "SELECT * FROM nested_objects WHERE nested(comment.likes) < 3", + * "filter": { + * "term": { + * "someField": "a" + * } + * } + * } + */ + int likesToCompare = 3; + String fieldToCompare = "a"; + SearchHits response = query(String.format("{\"query\":\"" + + "SELECT * " + + "FROM %s " + + "WHERE nested(comment.likes) < %s\"," + + "\"filter\":{\"term\":{\"someField\":\"%s\"}}}", + TestsConstants.TEST_INDEX_NESTED_TYPE, likesToCompare, fieldToCompare)); + SearchHit[] hits = response.getHits(); + for (SearchHit hit : hits) { + int likes = (int) ((Map) hit.getSourceAsMap().get("comment")).get("likes"); + String someField = hit.getSourceAsMap().get("someField").toString(); + assertThat(likes, lessThan(likesToCompare)); + assertThat(someField, equalTo(fieldToCompare)); + } + } + + @Test + public void searchWithNestedFilter() throws IOException { + /* + * Human readable format of the request defined below: + * { + * "query": "SELECT * FROM nested_objects WHERE nested(comment.likes) > 1", + * "filter": { + * "nested": { + * "path": "comment", + * "query": { + * "bool": { + * "must": { + * "term": { + * "comment.data": "aa" + * } + * } + * } + * } + * } + * } + * } + */ + int likesToCompare = 1; + String dataToCompare = "aa"; + SearchHits response = query(String.format("{\"query\":\"" + + "SELECT * " + + "FROM %s " + + "WHERE nested(comment.likes) > %s\"," + + "\"filter\":{\"nested\":{\"path\":\"comment\"," + + "\"query\":{\"bool\":{\"must\":{\"term\":{\"comment.data\":\"%s\"}}}}}}}", + TestsConstants.TEST_INDEX_NESTED_TYPE, likesToCompare, dataToCompare)); + SearchHit[] hits = response.getHits(); + for (SearchHit hit : hits) { + int likes = (int) ((Map) hit.getSourceAsMap().get("comment")).get("likes"); + String data = ((Map) hit.getSourceAsMap().get("comment")).get("data").toString(); + assertThat(likes, greaterThan(likesToCompare)); + assertThat(data, anyOf(equalTo(dataToCompare), equalTo("[aa, bb]"))); + } + } + + private SearchHits query(String request) throws IOException { + final JSONObject jsonObject = executeRequest(request); + + final XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser( + NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, + jsonObject.toString()); + return SearchResponse.fromXContent(parser).getHits(); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/JdbcTestIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/JdbcTestIT.java new file mode 100644 index 0000000000..90beb2dece --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/JdbcTestIT.java @@ -0,0 +1,175 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Assert; +import org.junit.Test; + +public class JdbcTestIT extends SQLIntegTestCase { + + @Override + protected void init() throws Exception { + loadIndex(Index.ONLINE); + loadIndex(Index.PEOPLE); + loadIndex(Index.ACCOUNT); + loadIndex(Index.WEBLOG); + } + + public void testPercentilesQuery() { + JSONObject response = executeJdbcRequest( + "SELECT percentiles(age, 25.0, 50.0, 75.0, 99.9) age_percentiles " + + "FROM elasticsearch-sql_test_index_people"); + + assertThat(response.getJSONArray("datarows").length(), equalTo(1)); + + JSONObject percentileRow = (JSONObject) response.query("/datarows/0/0"); + + assertThat(percentileRow.getDouble("25.0"), equalTo(31.5)); + assertThat(percentileRow.getDouble("50.0"), equalTo(33.5)); + assertThat(percentileRow.getDouble("75.0"), equalTo(36.5)); + assertThat(percentileRow.getDouble("99.9"), equalTo(39.0)); + } + + public void testDateTimeInQuery() { + JSONObject response = executeJdbcRequest( + "SELECT date_format(insert_time, 'dd-MM-YYYY') " + + "FROM elasticsearch-sql_test_index_online " + + "ORDER BY date_format(insert_time, 'dd-MM-YYYY') " + + "LIMIT 1" + ); + + assertThat( + response.getJSONArray("datarows") + .getJSONArray(0) + .getString(0), + equalTo("17-08-2014")); + } + + public void testDivisionInQuery() { + JSONObject response = executeJdbcRequest( + "SELECT all_client/10 from elasticsearch-sql_test_index_online ORDER BY all_client/10 desc limit 1"); + + assertThat( + response.getJSONArray("datarows") + .getJSONArray(0) + .getDouble(0), + equalTo(16827.0)); + } + + public void testGroupByInQuery() { + JSONObject response = executeJdbcRequest( + "SELECT date_format(insert_time, 'YYYY-MM-dd'), COUNT(*) " + + "FROM elasticsearch-sql_test_index_online " + + "GROUP BY date_format(insert_time, 'YYYY-MM-dd')" + ); + + assertThat(response.getJSONArray("schema").length(), equalTo(2)); + assertThat(response.getJSONArray("datarows").length(), equalTo(8)); + } + + private JSONObject executeJdbcRequest(String query) { + return new JSONObject(executeQuery(query, "jdbc")); + } + + @Test + public void numberOperatorNameCaseInsensitiveTest() { + assertSchemaContains( + executeQuery("SELECT ABS(age) FROM elasticsearch-sql_test_index_account " + + "WHERE age IS NOT NULL ORDER BY age LIMIT 5", "jdbc"), + "ABS(age)" + ); + } + + @Test + public void trigFunctionNameCaseInsensitiveTest() { + assertSchemaContains( + executeQuery("SELECT Cos(age) FROM elasticsearch-sql_test_index_account " + + "WHERE age is NOT NULL ORDER BY age LIMIT 5", "jdbc"), + "Cos(age)" + ); + } + + @Test + public void stringOperatorNameCaseInsensitiveTest() { + assertSchemaContains( + executeQuery("SELECT SubStrinG(lastname, 0, 2) FROM elasticsearch-sql_test_index_account " + + "ORDER BY age LIMIT 5", "jdbc"), + "SubStrinG(lastname, 0, 2)" + ); + } + + @Test + public void dateFunctionNameCaseInsensitiveTest() { + assertTrue( + executeQuery( + "SELECT DATE_FORMAT(insert_time, 'yyyy-MM-dd', 'UTC') FROM elasticsearch-sql_test_index_online " + + "WHERE date_FORMAT(insert_time, 'yyyy-MM-dd', 'UTC') > '2014-01-01' " + + "GROUP BY DAte_format(insert_time, 'yyyy-MM-dd', 'UTC') " + + "ORDER BY date_forMAT(insert_time, 'yyyy-MM-dd', 'UTC')", "jdbc").equalsIgnoreCase( + executeQuery( + "SELECT date_format(insert_time, 'yyyy-MM-dd', 'UTC') FROM elasticsearch-sql_test_index_online " + + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') > '2014-01-01' " + + "GROUP BY date_format(insert_time, 'yyyy-MM-dd', 'UTC') " + + "ORDER BY date_format(insert_time, 'yyyy-MM-dd', 'UTC')", "jdbc") + ) + ); + } + + @Test + public void ipTypeShouldPassJdbcFormatter() { + assertThat( + executeQuery("SELECT host AS hostIP FROM " + TestsConstants.TEST_INDEX_WEBLOG + + " ORDER BY hostIP", "jdbc"), + containsString("\"type\": \"ip\"") + ); + } + + @Test + public void functionWithoutAliasShouldHaveEntireFunctionAsNameInSchema() { + assertThat( + executeQuery("SELECT substring(lastname, 1, 2) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + + " ORDER BY substring(lastname, 1, 2)", "jdbc"), + containsString("\"name\": \"substring(lastname, 1, 2)\"") + ); + } + + @Test + public void functionWithAliasShouldHaveAliasAsNameInSchema() { + assertThat( + executeQuery("SELECT substring(lastname, 1, 2) AS substring FROM " + + TestsConstants.TEST_INDEX_ACCOUNT + " ORDER BY substring", "jdbc"), + containsString("\"name\": \"substring\"") + ); + } + + private void assertSchemaContains(String actualResponse, String expected) { + JSONArray schema = new JSONObject(actualResponse).optJSONArray("schema"); + for (Object nameTypePair : schema) { + String actual = ((JSONObject) nameTypePair).getString("name"); + if (expected.equals(actual)) { + return; + } + } + Assert.fail("Expected field name [" + expected + "] is not found in response schema: " + + actualResponse); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/JoinAliasWriterRuleIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/JoinAliasWriterRuleIT.java new file mode 100644 index 0000000000..86a0db6ccf --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/JoinAliasWriterRuleIT.java @@ -0,0 +1,261 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static org.hamcrest.Matchers.equalTo; + +import java.io.IOException; +import org.elasticsearch.client.ResponseException; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +/** + * Test cases for writing missing join table aliases. + */ +public class JoinAliasWriterRuleIT extends SQLIntegTestCase { + + @Rule + public ExpectedException exception = ExpectedException.none(); + + protected void init() throws Exception { + loadIndex(Index.ORDER); // elasticsearch-sql_test_index_order + loadIndex(Index.BANK); // elasticsearch-sql_test_index_bank + loadIndex(Index.BANK_TWO); // elasticsearch-sql_test_index_bank_two + } + + @Test + public void noTableAliasNoCommonColumns() throws IOException { + sameExplain( + query( + "SELECT id, firstname", + "FROM elasticsearch-sql_test_index_order", + "INNER JOIN elasticsearch-sql_test_index_bank ", + "ON name = firstname WHERE state = 'WA' OR id < 7"), + query( + "SELECT elasticsearch-sql_test_index_order_0.id, elasticsearch-sql_test_index_bank_1.firstname ", + "FROM elasticsearch-sql_test_index_order elasticsearch-sql_test_index_order_0 ", + "INNER JOIN elasticsearch-sql_test_index_bank elasticsearch-sql_test_index_bank_1 ", + "ON elasticsearch-sql_test_index_order_0.name = elasticsearch-sql_test_index_bank_1.firstname ", + "WHERE elasticsearch-sql_test_index_bank_1.state = 'WA' OR elasticsearch-sql_test_index_order_0.id < 7") + ); + } + + @Test + public void oneTableAliasNoCommonColumns() throws IOException { + sameExplain( + query( + "SELECT id, firstname ", + "FROM elasticsearch-sql_test_index_order a ", + "INNER JOIN elasticsearch-sql_test_index_bank ", + "ON name = firstname WHERE state = 'WA' OR id < 7"), + query( + "SELECT a.id, elasticsearch-sql_test_index_bank_0.firstname ", + "FROM elasticsearch-sql_test_index_order a ", + "INNER JOIN elasticsearch-sql_test_index_bank elasticsearch-sql_test_index_bank_0 ", + "ON a.name = elasticsearch-sql_test_index_bank_0.firstname ", + "WHERE elasticsearch-sql_test_index_bank_0.state = 'WA' OR a.id < 7") + ); + } + + @Test + public void bothTableAliasNoCommonColumns() throws IOException { + sameExplain( + query( + "SELECT id, firstname ", + "FROM elasticsearch-sql_test_index_order a ", + "INNER JOIN elasticsearch-sql_test_index_bank b ", + "ON name = firstname WHERE state = 'WA' OR id < 7 "), + query( + "SELECT a.id, b.firstname ", + "FROM elasticsearch-sql_test_index_order a ", + "INNER JOIN elasticsearch-sql_test_index_bank b ", + "ON a.name = b.firstname ", + "WHERE b.state = 'WA' OR a.id < 7 ") + ); + } + + @Test + public void tableNamesWithTypeName() throws IOException { + sameExplain( + query( + "SELECT id, firstname ", + "FROM elasticsearch-sql_test_index_order/_doc ", + "INNER JOIN elasticsearch-sql_test_index_bank/account ", + "ON name = firstname WHERE state = 'WA' OR id < 7"), + query( + "SELECT elasticsearch-sql_test_index_order_0.id, elasticsearch-sql_test_index_bank_1.firstname ", + "FROM elasticsearch-sql_test_index_order/_doc elasticsearch-sql_test_index_order_0 ", + "INNER JOIN elasticsearch-sql_test_index_bank/_account elasticsearch-sql_test_index_bank_1 ", + "ON elasticsearch-sql_test_index_order_0.name = elasticsearch-sql_test_index_bank_1.firstname ", + "WHERE elasticsearch-sql_test_index_bank_1.state = 'WA' OR elasticsearch-sql_test_index_order_0.id < 7") + ); + } + + @Ignore + @Test + public void tableNamesWithTypeNameExplicitTableAlias() throws IOException { + sameExplain( + query( + "SELECT id, firstname ", + "FROM elasticsearch-sql_test_index_order/_doc a ", + "INNER JOIN elasticsearch-sql_test_index_bank/account b ", + "ON name = firstname WHERE state = 'WA' OR id < 7"), + query( + "SELECT a.id, b.firstname ", + "FROM elasticsearch-sql_test_index_order a ", + "INNER JOIN elasticsearch-sql_test_index_bank b ", + "ON a.name = b.firstname ", + "WHERE b.state = 'WA' OR a.id < 7") + ); + } + + @Test + public void actualTableNameAsAliasOnColumnFields() throws IOException { + sameExplain( + query( + "SELECT elasticsearch-sql_test_index_order.id, b.firstname ", + "FROM elasticsearch-sql_test_index_order ", + "INNER JOIN elasticsearch-sql_test_index_bank b ", + "ON elasticsearch-sql_test_index_order.name = firstname WHERE state = 'WA' OR id < 7"), + query( + "SELECT elasticsearch-sql_test_index_order_0.id, b.firstname ", + "FROM elasticsearch-sql_test_index_order elasticsearch-sql_test_index_order_0 ", + "INNER JOIN elasticsearch-sql_test_index_bank b ", + "ON elasticsearch-sql_test_index_order_0.name = b.firstname ", + "WHERE b.state = 'WA' OR elasticsearch-sql_test_index_order_0.id < 7") + ); + } + + @Test + public void actualTableNameAsAliasOnColumnFieldsTwo() throws IOException { + sameExplain( + query( + "SELECT elasticsearch-sql_test_index_order.id, elasticsearch-sql_test_index_bank.firstname ", + "FROM elasticsearch-sql_test_index_order ", + "INNER JOIN elasticsearch-sql_test_index_bank ", + "ON elasticsearch-sql_test_index_order.name = firstname ", + "WHERE elasticsearch-sql_test_index_bank.state = 'WA' OR id < 7"), + query( + "SELECT elasticsearch-sql_test_index_order_0.id, elasticsearch-sql_test_index_bank_1.firstname ", + "FROM elasticsearch-sql_test_index_order elasticsearch-sql_test_index_order_0 ", + "INNER JOIN elasticsearch-sql_test_index_bank elasticsearch-sql_test_index_bank_1", + "ON elasticsearch-sql_test_index_order_0.name = elasticsearch-sql_test_index_bank_1.firstname ", + "WHERE elasticsearch-sql_test_index_bank_1.state = 'WA' OR elasticsearch-sql_test_index_order_0.id < 7") + ); + } + + @Test + public void columnsWithTableAliasNotAffected() throws IOException { + sameExplain( + query( + "SELECT a.id, firstname ", + "FROM elasticsearch-sql_test_index_order a ", + "INNER JOIN elasticsearch-sql_test_index_bank b ", + "ON name = b.firstname WHERE state = 'WA' OR a.id < 7"), + query( + "SELECT a.id, b.firstname ", + "FROM elasticsearch-sql_test_index_order a ", + "INNER JOIN elasticsearch-sql_test_index_bank b ", + "ON a.name = b.firstname ", + "WHERE b.state = 'WA' OR a.id < 7") + ); + } + + @Test + public void commonColumnWithoutTableAliasDifferentTables() throws IOException { + exception.expect(ResponseException.class); + exception.expectMessage("Field name [firstname] is ambiguous"); + String explain = explainQuery(query( + "SELECT firstname, lastname ", + "FROM elasticsearch-sql_test_index_bank ", + "LEFT JOIN elasticsearch-sql_test_index_bank_two ", + "ON firstname = lastname WHERE state = 'VA' " + )); + } + + @Test + public void sameTablesNoAliasAndNoAliasOnColumns() throws IOException { + exception.expect(ResponseException.class); + exception.expectMessage("Not unique table/alias: [elasticsearch-sql_test_index_bank]"); + String explain = explainQuery(query( + "SELECT firstname, lastname ", + "FROM elasticsearch-sql_test_index_bank ", + "LEFT JOIN elasticsearch-sql_test_index_bank ", + "ON firstname = lastname WHERE state = 'VA' " + )); + } + + @Test + public void sameTablesNoAliasWithTableNameAsAliasOnColumns() throws IOException { + exception.expect(ResponseException.class); + exception.expectMessage("Not unique table/alias: [elasticsearch-sql_test_index_bank]"); + String explain = explainQuery(query( + "SELECT elasticsearch-sql_test_index_bank.firstname", + "FROM elasticsearch-sql_test_index_bank ", + "JOIN elasticsearch-sql_test_index_bank ", + "ON elasticsearch-sql_test_index_bank.firstname = elasticsearch-sql_test_index_bank.lastname" + )); + } + + @Test + public void sameTablesWithExplicitAliasOnFirst() throws IOException { + sameExplain( + query( + "SELECT elasticsearch-sql_test_index_bank.firstname, a.lastname ", + "FROM elasticsearch-sql_test_index_bank a", + "JOIN elasticsearch-sql_test_index_bank ", + "ON elasticsearch-sql_test_index_bank.firstname = a.lastname " + ), + query( + "SELECT elasticsearch-sql_test_index_bank_0.firstname, a.lastname ", + "FROM elasticsearch-sql_test_index_bank a", + "JOIN elasticsearch-sql_test_index_bank elasticsearch-sql_test_index_bank_0", + "ON elasticsearch-sql_test_index_bank_0.firstname = a.lastname " + ) + + ); + } + + @Test + public void sameTablesWithExplicitAliasOnSecond() throws IOException { + sameExplain( + query( + "SELECT elasticsearch-sql_test_index_bank.firstname, a.lastname ", + "FROM elasticsearch-sql_test_index_bank ", + "JOIN elasticsearch-sql_test_index_bank a", + "ON elasticsearch-sql_test_index_bank.firstname = a.lastname " + ), + query( + "SELECT elasticsearch-sql_test_index_bank_0.firstname, a.lastname ", + "FROM elasticsearch-sql_test_index_bank elasticsearch-sql_test_index_bank_0", + "JOIN elasticsearch-sql_test_index_bank a", + "ON elasticsearch-sql_test_index_bank_0.firstname = a.lastname " + ) + + ); + } + + private void sameExplain(String actualQuery, String expectedQuery) throws IOException { + assertThat(explainQuery(actualQuery), equalTo(explainQuery(expectedQuery))); + } + + private String query(String... statements) { + return String.join(" ", statements); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/JoinIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/JoinIT.java new file mode 100644 index 0000000000..3233f08f89 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/JoinIT.java @@ -0,0 +1,1025 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_DOG; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_DOG2; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_GAME_OF_THRONES; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_LOCATION; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_LOCATION2; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_PEOPLE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_PEOPLE2; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.stream.IntStream; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + +public class JoinIT extends SQLIntegTestCase { + + private static final String USE_NL_HINT = " /*! USE_NL*/"; + + @Override + protected void init() throws Exception { + + loadIndex(Index.DOG); + loadIndex(Index.DOGS2); + loadIndex(Index.PEOPLE); + loadIndex(Index.PEOPLE2); + loadIndex(Index.GAME_OF_THRONES); + loadIndex(Index.LOCATION); + loadIndex(Index.LOCATION_TWO); + loadIndex(Index.ACCOUNT); + } + + @Test + public void joinParseCheckSelectedFieldsSplitHASH() throws IOException { + joinParseCheckSelectedFieldsSplit(false); + } + + @Test + public void joinParseCheckSelectedFieldsSplitNL() throws IOException { + joinParseCheckSelectedFieldsSplit(true); + } + + @Test + public void joinParseWithHintsCheckSelectedFieldsSplitHASH() throws IOException { + + String query = String.format(Locale.ROOT, "SELECT /*! HASH_WITH_TERMS_FILTER*/ " + + "a.firstname ,a.lastname, a.gender ,d.dog_name FROM %s a JOIN %s d " + + "ON d.holdersName = a.firstname WHERE (a.age > 10 OR a.balance > 2000) AND d.age > 1", + TEST_INDEX_PEOPLE, TEST_INDEX_DOG); + + JSONObject result = executeQuery(query); + verifyJoinParseCheckSelectedFieldsSplitResult(result, false); + + String explanation = explainQuery(query); + Assert.assertThat(explanation, containsString("holdersName")); + + // TODO: figure out why explain does not show results from first query in term filter and + // fix either the test or the code. + //Arrays.asList("daenerys","nanette","virginia","aurelia","mcgee","hattie","elinor","burton").forEach(name -> { + // Assert.assertThat(explanation, containsString(name)); + //}); + } + + @Test + public void joinWithNoWhereButWithConditionHash() throws IOException { + + joinWithNoWhereButWithCondition(false); + } + + @Test + public void joinWithNoWhereButWithConditionNL() throws IOException { + + joinWithNoWhereButWithCondition(true); + } + + @Test + public void joinWithStarHASH() throws IOException { + + String query = String.format(Locale.ROOT, "SELECT * FROM %1$s c " + + "JOIN %1$s h ON h.hname = c.house ", TEST_INDEX_GAME_OF_THRONES); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + + Assert.assertEquals(4, hits.length()); + String house = (String) hits.query("/0/_source/c.house"); + Assert.assertThat(house, anyOf(equalTo("Targaryen"), equalTo("Stark"), equalTo("Lannister"))); + String houseName = (String) hits.query("/0/_source/h.hname"); + Assert.assertEquals(house, houseName); + } + + @Test + public void joinNoConditionButWithWhereHASH() throws IOException { + + joinNoConditionButWithWhere(false); + } + + @Test + public void joinNoConditionButWithWhereNL() throws IOException { + + joinNoConditionButWithWhere(true); + } + + @Test + public void joinNoConditionAndNoWhereHASH() throws IOException { + + joinNoConditionAndNoWhere(false); + } + + @Test + public void joinNoConditionAndNoWhereNL() throws IOException { + + joinNoConditionAndNoWhere(true); + } + + @Test + public void joinNoConditionAndNoWhereWithTotalLimitHASH() throws IOException { + + joinNoConditionAndNoWhereWithTotalLimit(false); + } + + @Test + public void joinNoConditionAndNoWhereWithTotalLimitNL() throws IOException { + + joinNoConditionAndNoWhereWithTotalLimit(true); + } + + @Test + public void joinWithNestedFieldsOnReturnHASH() throws IOException { + + joinWithNestedFieldsOnReturn(false); + } + + @Test + public void joinWithNestedFieldsOnReturnNL() throws IOException { + + joinWithNestedFieldsOnReturn(true); + } + + @Test + public void joinWithAllAliasOnReturnHASH() throws IOException { + + joinWithAllAliasOnReturn(false); + } + + @Test + public void joinWithAllAliasOnReturnNL() throws IOException { + + joinWithAllAliasOnReturn(true); + } + + @Test + public void joinWithSomeAliasOnReturnHASH() throws IOException { + + joinWithSomeAliasOnReturn(false); + } + + @Test + public void joinWithSomeAliasOnReturnNL() throws IOException { + + joinWithSomeAliasOnReturn(true); + } + + @Test + public void joinWithNestedFieldsOnComparisonAndOnReturnHASH() throws IOException { + + joinWithNestedFieldsOnComparisonAndOnReturn(false); + } + + @Test + public void joinWithNestedFieldsOnComparisonAndOnReturnNL() throws IOException { + + joinWithNestedFieldsOnComparisonAndOnReturn(true); + } + + @Test + public void testLeftJoinHASH() throws IOException { + + testLeftJoin(false); + } + + @Test + public void testLeftJoinNL() throws IOException { + + testLeftJoin(true); + } + + @Test + public void hintLimits_firstLimitSecondNullHASH() throws IOException { + + hintLimits_firstLimitSecondNull(false); + } + + @Test + public void hintLimits_firstLimitSecondNullNL() throws IOException { + + hintLimits_firstLimitSecondNull(true); + } + + @Test + public void hintLimits_firstLimitSecondLimitHASH() throws IOException { + + hintLimits_firstLimitSecondLimit(false); + } + + @Test + public void hintLimits_firstLimitSecondLimitNL() throws IOException { + + hintLimits_firstLimitSecondLimit(true); + } + + @Ignore("Join limit hint is deprecated and easily broken due to limit on unsorted records") + @Test + public void hintLimits_firstLimitSecondLimitOnlyOneNL() throws IOException { + + hintLimits_firstLimitSecondLimitOnlyOne(true); + } + + @Ignore("Join limit hint is deprecated and easily broken due to limit on unsorted records") + @Test + public void hintLimits_firstLimitSecondLimitOnlyOneHASH() throws IOException { + + hintLimits_firstLimitSecondLimitOnlyOne(false); + } + + @Test + public void hintLimits_firstNullSecondLimitHASH() throws IOException { + + hintLimits_firstNullSecondLimit(false); + } + + @Test + public void hintLimits_firstNullSecondLimitNL() throws IOException { + + hintLimits_firstNullSecondLimit(true); + } + + @Test + public void testLeftJoinWithLimitHASH() throws IOException { + + testLeftJoinWithLimit(false); + } + + @Test + public void testLeftJoinWithLimitNL() throws IOException { + + testLeftJoinWithLimit(true); + } + + @Test + public void hintMultiSearchCanRunFewTimesNL() throws IOException { + + String query = String.format(Locale.ROOT, "SELECT /*! USE_NL*/ /*! NL_MULTISEARCH_SIZE(2)*/ " + + "c.name.firstname,c.parents.father,h.hname,h.words FROM %1$s c " + + "JOIN %1$s h", TEST_INDEX_GAME_OF_THRONES); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + Assert.assertThat(hits.length(), equalTo(42)); + } + + @Test + public void joinWithGeoIntersectNL() throws IOException { + + String query = String.format(Locale.ROOT, "SELECT p1.description,p2.description " + + "FROM %s p1 JOIN %s p2 ON GEO_INTERSECTS(p2.place,p1.place)", + TEST_INDEX_LOCATION, TEST_INDEX_LOCATION2); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + Assert.assertThat(hits.length(), equalTo(2)); + + Assert.assertThat(hits.query("/0/_source/p2.description"), equalTo("squareRelated")); + Assert.assertThat(hits.query("/1/_source/p2.description"), equalTo("squareRelated")); + } + + // TODO: resolve issue #25 in github and update the test if needed + // TODO: this test causes the in-memory node to fail/crash + @Ignore + @Test + public void joinWithInQuery() throws IOException { + + //TODO: Either change the ON condition field to keyword or create a different subquery + String query = String.format(Locale.ROOT, "SELECT c.gender,c.name.firstname,h.hname,h.words " + + "FROM %1$s c JOIN %1$s h ON h.hname = c.house " + + "WHERE c.name.firstname IN (SELECT holdersName FROM %2$s)", + TEST_INDEX_GAME_OF_THRONES, TEST_INDEX_DOG); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + Assert.assertThat(hits.length(), equalTo(1)); + Assert.assertThat(hits.query("/0/_source/c.name.firstname"), equalTo("Daenerys")); + } + + @Test + public void joinWithOrHASH() throws IOException { + + joinWithOr(false); + } + + @Test + public void joinWithOrNL() throws IOException { + + joinWithOr(true); + } + + @Ignore // TODO: explanation does not have the terms section + @Test + public void joinWithOrWithTermsFilterOpt() throws IOException { + + String query = String.format(Locale.ROOT, "SELECT /*! HASH_WITH_TERMS_FILTER*/ " + + "d.dog_name,c.name.firstname FROM %s c " + + "JOIN %s d ON d.holdersName = c.name.firstname OR d.age = c.name.ofHisName", + TEST_INDEX_GAME_OF_THRONES, TEST_INDEX_DOG); + + executeQuery(query); + String explanation = explainQuery(query); + + Assert.assertTrue(containsTerm(explanation, "holdersName")); + Assert.assertTrue(containsTerm(explanation, "age")); + + Arrays.asList("daenerys", "brandon", "eddard", "jaime").forEach( + name -> Assert.assertTrue(explanation.contains(name)) + ); + } + + @Test + public void joinWithOrderbyFirstTableHASH() throws IOException { + + joinWithOrderFirstTable(false); + } + + @Test + public void joinWithOrderbyFirstTableNL() throws IOException { + + joinWithOrderFirstTable(true); + } + + @Test + public void joinWithAllFromSecondTableHASH() throws IOException { + + joinWithAllFromSecondTable(false); + } + + @Test + public void joinWithAllFromSecondTableNL() throws IOException { + + joinWithAllFromSecondTable(true); + } + + @Test + public void joinWithAllFromFirstTableHASH() throws IOException { + + joinWithAllFromFirstTable(false); + } + + @Test + public void joinWithAllFromFirstTableNL() throws IOException { + + joinWithAllFromFirstTable(true); + } + + @Test + public void leftJoinWithAllFromSecondTableHASH() throws IOException { + + leftJoinWithAllFromSecondTable(false); + } + + @Test + public void leftJoinWithAllFromSecondTableNL() throws IOException { + + leftJoinWithAllFromSecondTable(true); + } + + @Test + public void joinParseCheckSelectedFieldsSplitNLConditionOrderEQ() throws IOException { + + final String query = String.format(Locale.ROOT, "SELECT /*! USE_NL*/ " + + "a.firstname, a.lastname, a.gender, d.dog_name FROM %s a JOIN %s d " + + "ON a.firstname = d.holdersName WHERE (a.age > 10 OR a.balance > 2000) AND d.age > 1", + TEST_INDEX_PEOPLE2, TEST_INDEX_DOG2); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + + Assert.assertThat(hits.length(), equalTo(2)); + + Map match1 = ImmutableMap.of( + "a.firstname", "Daenerys", + "a.lastname", "Targaryen", + "a.gender", "M", + "d.dog_name", "rex"); + Map match2 = ImmutableMap.of( + "a.firstname", "Hattie", + "a.lastname", "Bond", + "a.gender", "M", + "d.dog_name", "snoopy"); + + Assert.assertTrue(hitsInclude(hits, match1)); + Assert.assertTrue(hitsInclude(hits, match2)); + } + + @Test + public void joinParseCheckSelectedFieldsSplitNLConditionOrderGT() throws IOException { + + final String query = String.format(Locale.ROOT, "SELECT /*! USE_NL*/ " + + "a.firstname, a.lastname, a.gender, d.firstname, d.age FROM " + + "%s a JOIN %s d on a.age < d.age " + + "WHERE (d.firstname = 'Lynn' OR d.firstname = 'Obrien') AND a.firstname = 'Mcgee'", + TEST_INDEX_PEOPLE, TEST_INDEX_ACCOUNT); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + + Assert.assertThat(hits.length(), equalTo(2)); + + Map oneMatch = ImmutableMap.of("a.firstname", "Mcgee", "a.lastname", "Mooney", + "a.gender", "M", "d.firstname", "Obrien", "d.age", 40); + Map secondMatch = ImmutableMap.of("a.firstname", "Mcgee", "a.lastname", "Mooney", + "a.gender", "M", "d.firstname", "Lynn", "d.age", 40); + + Assert.assertTrue(hitsInclude(hits, oneMatch)); + Assert.assertTrue(hitsInclude(hits, secondMatch)); + } + + @Test + public void joinParseCheckSelectedFieldsSplitNLConditionOrderLT() throws IOException { + + final String query = String.format(Locale.ROOT, "SELECT /*! USE_NL*/ " + + "a.firstname, a.lastname, a.gender, d.firstname, d.age FROM " + + "%s a JOIN %s d on a.age > d.age " + + "WHERE (d.firstname = 'Sandoval' OR d.firstname = 'Hewitt') AND a.firstname = 'Fulton'", + TEST_INDEX_PEOPLE, TEST_INDEX_ACCOUNT); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + + Assert.assertThat(hits.length(), equalTo(2)); + + Map oneMatch = ImmutableMap.of("a.firstname", "Fulton", "a.lastname", "Holt", + "a.gender", "F", "d.firstname", "Sandoval", "d.age", 22); + Map secondMatch = ImmutableMap.of("a.firstname", "Fulton", "a.lastname", "Holt", + "a.gender", "F", "d.firstname", "Hewitt", "d.age", 22); + + Assert.assertTrue(hitsInclude(hits, oneMatch)); + Assert.assertTrue(hitsInclude(hits, secondMatch)); + } + + @Test + public void leftJoinNLWithNullInCondition() throws IOException { + + joinWithNullInCondition(true, "LEFT", "OR", "OR", 7); + } + + @Test + public void leftJoinNLWithNullInCondition1() throws IOException { + + joinWithNullInCondition(true, "LEFT", "OR", "AND", 7); + } + + @Test + public void leftJoinNLWithNullInCondition2() throws IOException { + + joinWithNullInCondition(true, "LEFT", "AND", "AND", 7); + } + + @Test + public void leftJoinNLWithNullInCondition3() throws IOException { + + joinWithNullInCondition(true, "LEFT", "AND", "OR", 7); + } + + @Test + public void innerJoinNLWithNullInCondition() throws IOException { + + joinWithNullInCondition(true, "", "OR", "OR", 0); + } + + @Test + public void innerJoinNLWithNullInCondition1() throws IOException { + + joinWithNullInCondition(true, "", "OR", "AND", 0); + } + + @Test + public void innerJoinNLWithNullInCondition2() throws IOException { + + joinWithNullInCondition(true, "", "AND", "AND", 0); + } + + @Test + public void innerJoinNLWithNullInCondition3() throws IOException { + + joinWithNullInCondition(true, "", "AND", "OR", 0); + } + + private void joinWithAllFromSecondTable(boolean useNestedLoops) throws IOException { + + final String hint = useNestedLoops ? USE_NL_HINT : ""; + final String query = String.format(Locale.ROOT, "SELECT%1$s c.name.firstname, d.* " + + "FROM %2$s c JOIN %2$s d ON d.hname = c.house", + hint, TEST_INDEX_GAME_OF_THRONES); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + + if (useNestedLoops) { + Assert.assertThat(hits.length(), equalTo(0)); + } else { + Assert.assertThat(hits.length(), equalTo(4)); + Assert.assertThat(hits.getJSONObject(0).getJSONObject("_source").length(), equalTo(5)); + } + } + + private void joinWithAllFromFirstTable(boolean useNestedLoops) throws IOException { + + final String hint = useNestedLoops ? USE_NL_HINT : ""; + final String query = String.format(Locale.ROOT, "SELECT%1$s c.name.firstname " + + "FROM %2$s d JOIN %2$s c ON c.house = d.hname", + hint, TEST_INDEX_GAME_OF_THRONES); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + + if (useNestedLoops) { + Assert.assertThat(hits.length(), equalTo(0)); + } else { + Assert.assertThat(hits.length(), equalTo(4)); + Assert.assertThat(hits.getJSONObject(0).getJSONObject("_source").length(), equalTo(1)); + } + } + + private void leftJoinWithAllFromSecondTable(boolean useNestedLoops) throws IOException { + + final String hint = useNestedLoops ? USE_NL_HINT : ""; + final String query = String.format(Locale.ROOT, "SELECT%1$s c.name.firstname, d.* " + + "FROM %2$s c LEFT JOIN %2$s d ON d.hname = c.house", + hint, TEST_INDEX_GAME_OF_THRONES); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + + Assert.assertThat(hits.length(), equalTo(7)); + + hits.forEach(hitObj -> { + JSONObject hit = (JSONObject) hitObj; + + Assert.assertThat(hit.getJSONObject("_source").length(), + equalTo(hit.getString("_id").endsWith("0") ? 1 : 5)); + }); + } + + private void joinParseCheckSelectedFieldsSplit(boolean useNestedLoops) throws IOException { + + final String hint = useNestedLoops ? USE_NL_HINT : ""; + String query = + String.format(Locale.ROOT, "SELECT%s a.firstname ,a.lastname,a.gender,d.dog_name " + + "FROM %s a JOIN %s d ON d.holdersName = a.firstname " + + "WHERE (a.age > 10 OR a.balance > 2000) AND d.age > 1", hint, TEST_INDEX_PEOPLE, + TEST_INDEX_DOG); + + JSONObject result = executeQuery(query); + verifyJoinParseCheckSelectedFieldsSplitResult(result, useNestedLoops); + } + + private void joinNoConditionButWithWhere(boolean useNestedLoops) throws IOException { + + final String hint = useNestedLoops ? USE_NL_HINT : ""; + String query = String.format(Locale.ROOT, "SELECT%s c.gender,h.hname,h.words FROM %2$s c " + + "JOIN %2$s h WHERE match_phrase(c.name.firstname, 'Daenerys')", + hint, TEST_INDEX_GAME_OF_THRONES); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + Assert.assertThat(hits.length(), equalTo(7)); + } + + private void joinNoConditionAndNoWhere(boolean useNestedLoops) throws IOException { + + final String hint = useNestedLoops ? USE_NL_HINT : ""; + String query = + String.format(Locale.ROOT, "SELECT%s c.name.firstname,c.parents.father,h.hname,h.words " + + "FROM %2$s c JOIN %2$s h", + hint, TEST_INDEX_GAME_OF_THRONES); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + Assert.assertThat(hits.length(), equalTo(49)); + } + + private void joinWithNoWhereButWithCondition(boolean useNestedLoops) throws IOException { + + final String hint = useNestedLoops ? USE_NL_HINT : ""; + String query = String.format(Locale.ROOT, "SELECT%s c.gender,h.hname,h.words " + + "FROM %2$s c JOIN %2$s h ON h.hname = c.house", + hint, TEST_INDEX_GAME_OF_THRONES); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + + Map someMatch = ImmutableMap.of( + "c.gender", "F", + "h.hname", "Targaryen", + "h.words", "fireAndBlood"); + + if (useNestedLoops) { + // TODO: should the NL result be different? + Assert.assertThat(hits.length(), equalTo(0)); + } else { + Assert.assertThat(hits.length(), equalTo(4)); + Assert.assertTrue(hitsInclude(hits, someMatch)); + } + } + + private void verifyJoinParseCheckSelectedFieldsSplitResult(JSONObject result, + boolean useNestedLoops) { + + Map match1 = ImmutableMap.of( + "a.firstname", "Daenerys", + "a.lastname", "Targaryen", + "a.gender", "M", + "d.dog_name", "rex"); + Map match2 = ImmutableMap.of( + "a.firstname", "Hattie", + "a.lastname", "Bond", + "a.gender", "M", + "d.dog_name", "snoopy"); + + JSONArray hits = getHits(result); + + if (useNestedLoops) { + //TODO: change field mapping in ON condition to keyword or change query to get result + // TODO: why does NL query return no results? + Assert.assertThat(hits.length(), equalTo(0)); + } else { + Assert.assertThat(hits.length(), equalTo(2)); + Assert.assertTrue(hitsInclude(hits, match1)); + Assert.assertTrue(hitsInclude(hits, match2)); + } + } + + private void joinNoConditionAndNoWhereWithTotalLimit(boolean useNestedLoops) throws IOException { + + final String hint = useNestedLoops ? USE_NL_HINT : ""; + String query = + String.format(Locale.ROOT, "SELECT%s c.name.firstname,c.parents.father,h.hname,h.words" + + " FROM %2$s c JOIN %2$s h LIMIT 9", + hint, TEST_INDEX_GAME_OF_THRONES); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + Assert.assertThat(hits.length(), equalTo(9)); + } + + private void joinWithNestedFieldsOnReturn(boolean useNestedLoops) throws IOException { + + final String hint = useNestedLoops ? USE_NL_HINT : ""; + String query = + String.format(Locale.ROOT, "SELECT%s c.name.firstname,c.parents.father,h.hname,h.words " + + "FROM %2$s c JOIN %2$s h ON h.hname = c.house " + + "WHERE match_phrase(c.name.firstname, 'Daenerys')", + hint, TEST_INDEX_GAME_OF_THRONES); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + final Map expectedMatch = ImmutableMap.of( + "c.name.firstname", "Daenerys", + "c.parents.father", "Aerys", + "h.hname", "Targaryen", + "h.words", "fireAndBlood"); + if (useNestedLoops) { + Assert.assertThat(hits.length(), equalTo(0)); + } else { + Assert.assertThat(hits.length(), equalTo(1)); + assertHitMatches(hits.getJSONObject(0), expectedMatch); + } + } + + private void joinWithAllAliasOnReturn(boolean useNestedLoops) throws IOException { + + final String hint = useNestedLoops ? USE_NL_HINT : ""; + String query = + String.format(Locale.ROOT, "SELECT%s c.name.firstname name,c.parents.father father," + + "h.hname house FROM %2$s c JOIN %2$s h ON h.hname = c.house " + + "WHERE match_phrase(c.name.firstname, 'Daenerys')", + hint, TEST_INDEX_GAME_OF_THRONES); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + final Map expectedMatch = ImmutableMap.of( + "name", "Daenerys", + "father", "Aerys", + "house", "Targaryen"); + + if (useNestedLoops) { + Assert.assertThat(hits.length(), equalTo(0)); + } else { + Assert.assertThat(hits.length(), equalTo(1)); + assertHitMatches(hits.getJSONObject(0), expectedMatch); + } + } + + private void joinWithSomeAliasOnReturn(boolean useNestedLoops) throws IOException { + + final String hint = useNestedLoops ? USE_NL_HINT : ""; + String query = + String.format(Locale.ROOT, "SELECT%s c.name.firstname ,c.parents.father father, " + + "h.hname house FROM %2$s c JOIN %2$s h ON h.hname = c.house " + + "WHERE match_phrase(c.name.firstname, 'Daenerys')", + hint, TEST_INDEX_GAME_OF_THRONES); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + final Map expectedMatch = ImmutableMap.of( + "c.name.firstname", "Daenerys", + "father", "Aerys", + "house", "Targaryen"); + + if (useNestedLoops) { + //TODO: Either change the ON condition field to keyword or create a different subquery + Assert.assertThat(hits.length(), equalTo(0)); + } else { + Assert.assertThat(hits.length(), equalTo(1)); + assertHitMatches(hits.getJSONObject(0), expectedMatch); + } + } + + private void joinWithNestedFieldsOnComparisonAndOnReturn(boolean useNestedLoops) + throws IOException { + + final String hint = useNestedLoops ? USE_NL_HINT : ""; + String query = + String.format(Locale.ROOT, "SELECT%s c.name.firstname,c.parents.father, h.hname,h.words " + + " FROM %2$s c JOIN %2$s h ON h.hname = c.name.lastname " + + "WHERE match_phrase(c.name.firstname, 'Daenerys')", + hint, TEST_INDEX_GAME_OF_THRONES); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + final Map expectedMatch = ImmutableMap.of( + "c.name.firstname", "Daenerys", + "c.parents.father", "Aerys", + "h.hname", "Targaryen", + "h.words", "fireAndBlood"); + + if (useNestedLoops) { + Assert.assertThat(hits.length(), equalTo(0)); + } else { + Assert.assertThat(hits.length(), equalTo(1)); + assertHitMatches(hits.getJSONObject(0), expectedMatch); + } + } + + private void testLeftJoin(boolean useNestedLoops) throws IOException { + + final String hint = useNestedLoops ? USE_NL_HINT : ""; + String query = String.format("SELECT%s c.name.firstname, f.name.firstname,f.name.lastname " + + "FROM %2$s c LEFT JOIN %2$s f " + + "ON f.name.firstname = c.parents.father", + hint, TEST_INDEX_GAME_OF_THRONES); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + Assert.assertThat(hits.length(), equalTo(7)); + + final Map firstMatch = new HashMap<>(); + firstMatch.put("c.name.firstname", "Daenerys"); + firstMatch.put("f.name.firstname", null); + firstMatch.put("f.name.lastname", null); + + final Map secondMatch = new HashMap<>(); + secondMatch.put("c.name.firstname", "Brandon"); + + if (useNestedLoops) { + secondMatch.put("f.name.firstname", null); + secondMatch.put("f.name.lastname", null); + } else { + secondMatch.put("f.name.firstname", "Eddard"); + secondMatch.put("f.name.lastname", "Stark"); + } + + Assert.assertTrue(hitsInclude(hits, firstMatch)); + Assert.assertTrue(hitsInclude(hits, secondMatch)); + } + + private void hintLimits_firstLimitSecondNull(boolean useNestedLoops) throws IOException { + + final String hint = useNestedLoops ? USE_NL_HINT : ""; + String query = String.format(Locale.ROOT, "SELECT%s /*! JOIN_TABLES_LIMIT(2,null) */ " + + "c.name.firstname,c.parents.father, h.hname,h.words " + + "FROM %2$s c JOIN %2$s h", + hint, TEST_INDEX_GAME_OF_THRONES); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + Assert.assertThat(hits.length(), equalTo(14)); + } + + private void hintLimits_firstLimitSecondLimit(boolean useNestedLoops) throws IOException { + + final String hint = useNestedLoops ? USE_NL_HINT : ""; + String query = String.format(Locale.ROOT, "SELECT%s /*! JOIN_TABLES_LIMIT(2,2) */ " + + "c.name.firstname,c.parents.father, h.hname,h.words FROM %2$s c " + + "JOIN %2$s h", hint, TEST_INDEX_GAME_OF_THRONES); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + Assert.assertThat(hits.length(), equalTo(4)); + } + + private void hintLimits_firstLimitSecondLimitOnlyOne(boolean useNestedLoops) throws IOException { + + final String hint = useNestedLoops ? USE_NL_HINT : ""; + String query = String.format(Locale.ROOT, "SELECT%s /*! JOIN_TABLES_LIMIT(3,1) */ " + + "c.name.firstname,c.parents.father , h.hname,h.words FROM %2$s h " + + "JOIN %2$s c ON c.name.lastname = h.hname", + hint, TEST_INDEX_GAME_OF_THRONES); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + Assert.assertThat(hits.length(), equalTo(0)); + } + + private void hintLimits_firstNullSecondLimit(boolean useNestedLoops) throws IOException { + + final String hint = useNestedLoops ? USE_NL_HINT : ""; + String query = String.format(Locale.ROOT, "SELECT%s /*! JOIN_TABLES_LIMIT(null,2) */ " + + "c.name.firstname,c.parents.father , h.hname,h.words FROM %2$s c " + + "JOIN %2$s h", hint, TEST_INDEX_GAME_OF_THRONES); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + Assert.assertThat(hits.length(), equalTo(14)); + } + + private void testLeftJoinWithLimit(boolean useNestedLoops) throws IOException { + + final String hint = useNestedLoops ? USE_NL_HINT : ""; + String query = String.format(Locale.ROOT, "SELECT%s /*! JOIN_TABLES_LIMIT(3,null) */ " + + "c.name.firstname, f.name.firstname,f.name.lastname FROM %2$s c " + + "LEFT JOIN %2$s f ON f.name.firstname = c.parents.father", + hint, TEST_INDEX_GAME_OF_THRONES); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + Assert.assertThat(hits.length(), equalTo(3)); + } + + private void joinWithOr(boolean useNestedLoops) throws IOException { + + final String hint = useNestedLoops ? USE_NL_HINT : ""; + String query = String.format(Locale.ROOT, "SELECT%s d.dog_name,c.name.firstname " + + "FROM %s c JOIN %s d " + + "ON d.holdersName = c.name.firstname OR d.age = c.name.ofHisName", + hint, TEST_INDEX_GAME_OF_THRONES, TEST_INDEX_DOG); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + + final Map firstMatch = ImmutableMap.of( + "c.name.firstname", "Daenerys", + "d.dog_name", "rex"); + final Map secondMatch = ImmutableMap.of( + "c.name.firstname", "Brandon", + "d.dog_name", "snoopy"); + + if (useNestedLoops) { + Assert.assertThat(hits.length(), equalTo(1)); + Assert.assertTrue("hits contains brandon", hitsInclude(hits, secondMatch)); + } else { + Assert.assertThat(hits.length(), equalTo(2)); + Assert.assertTrue("hits contains daenerys", hitsInclude(hits, firstMatch)); + Assert.assertTrue("hits contains brandon", hitsInclude(hits, secondMatch)); + } + } + + private void joinWithOrderFirstTable(boolean useNestedLoops) throws IOException { + + final String hint = useNestedLoops ? USE_NL_HINT : ""; + String query = String.format(Locale.ROOT, "SELECT%s c.name.firstname,d.words " + + "FROM %2$s c JOIN %2$s d ON d.hname = c.house " + + "ORDER BY c.name.firstname", + hint, TEST_INDEX_GAME_OF_THRONES); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + + if (useNestedLoops) { + Assert.assertThat(hits.length(), equalTo(0)); + } else { + + Assert.assertThat(hits.length(), equalTo(4)); + + String[] expectedNames = {"Brandon", "Daenerys", "Eddard", "Jaime"}; + + IntStream.rangeClosed(0, 3).forEach(i -> { + String firstnamePath = String.format(Locale.ROOT, "/%d/_source/c.name.firstname", i); + Assert.assertThat(hits.query(firstnamePath), equalTo(expectedNames[i])); + }); + } + } + + private boolean containsTerm(final String explainedQuery, final String termName) { + + return Pattern.compile( + Pattern.quote("\"terms\":{") + + ".*" + + Pattern.quote("\"" + termName + "\":[") + ) + .matcher(explainedQuery.replaceAll("\\s+", "")) + .find(); + } + + private void joinWithNullInCondition(boolean useNestedLoops, String left, + String oper1, String oper2, int expectedNum) + throws IOException { + + final String hint = useNestedLoops ? USE_NL_HINT : ""; + String query = + String.format(Locale.ROOT, "SELECT%s c.name.firstname,c.parents.father,c.hname," + + "f.name.firstname,f.house,f.hname FROM %s c " + + "%s JOIN %s f ON f.name.firstname = c.parents.father " + + "%s f.house = c.hname %s f.house = c.name.firstname", + hint, TEST_INDEX_GAME_OF_THRONES, left, TEST_INDEX_GAME_OF_THRONES, oper1, oper2); + + JSONObject result = executeQuery(query); + JSONArray hits = getHits(result); + + if (useNestedLoops) { + Assert.assertThat(hits.length(), equalTo(expectedNum)); + } else { + // This branch is reserved for hash join, no test currently enters it. + // Making it fail, so that whenever tests are added for hash join, + // this method gets properly updated. + Assert.fail(); + } + } + + private boolean hitsInclude(final JSONArray actualHits, Map expectedSourceValues) { + + for (final Object hitObj : actualHits.toList()) { + + final Map hit = uncheckedGetMap(hitObj); + if (hitMatches(hit, expectedSourceValues)) { + return true; + } + } + + return false; + } + + private void assertHitMatches(final JSONObject actualHit, + final Map expectedSourceValues) { + + final JSONObject src = actualHit.getJSONObject("_source"); + Assert.assertThat(src.length(), equalTo(expectedSourceValues.size())); + src.keySet().forEach(key -> { + Assert.assertTrue(expectedSourceValues.containsKey(key)); + Object value = src.get(key); + Assert.assertThat(value, equalTo(expectedSourceValues.get(key))); + }); + } + + private boolean hitMatches(final Map actualHit, + final Map expectedSourceValues) { + + final Map src = uncheckedGetMap(actualHit.get("_source")); + + if (src.size() != expectedSourceValues.size()) { + return false; + } + + for (final String key : src.keySet()) { + + if (!expectedSourceValues.containsKey(key)) { + return false; + } + + Object actualValue = src.get(key); + Object expectedValue = expectedSourceValues.get(key); + if ((actualValue == null && expectedValue != null) || + (actualValue != null && expectedValue == null)) { + return false; + } else if (actualValue != null && !actualValue.equals(expectedValue)) { + return false; + } + } + + return true; + } + + @SuppressWarnings("unchecked") + private Map uncheckedGetMap(final Object mapObject) { + return (Map) mapObject; + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MathFunctionsIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MathFunctionsIT.java new file mode 100644 index 0000000000..b51bc6475b --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MathFunctionsIT.java @@ -0,0 +1,265 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; + +import java.io.IOException; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.search.SearchHit; +import org.junit.Test; + +public class MathFunctionsIT extends SQLIntegTestCase { + + private static final String FROM = "FROM " + TestsConstants.TEST_INDEX_ACCOUNT; + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + } + + @Test + public void lowerCaseFunctionCall() throws IOException { + SearchHit[] hits = query( + "SELECT abs(age - 100) AS abs" + ); + for (SearchHit hit : hits) { + double abs = (double) getField(hit, "abs"); + assertThat(abs, greaterThanOrEqualTo(0.0)); + } + } + + @Test + public void upperCaseFunctionCall() throws IOException { + SearchHit[] hits = query( + "SELECT ABS(age - 100) AS abs" + ); + for (SearchHit hit : hits) { + double abs = (double) getField(hit, "abs"); + assertThat(abs, greaterThanOrEqualTo(0.0)); + } + } + + @Test + public void eulersNumber() throws IOException { + SearchHit[] hits = query( + "SELECT E() AS e" + ); + double e = (double) getField(hits[0], "e"); + assertThat(e, equalTo(Math.E)); + } + + @Test + public void pi() throws IOException { + SearchHit[] hits = query( + "SELECT PI() AS pi" + ); + double pi = (double) getField(hits[0], "pi"); + assertThat(pi, equalTo(Math.PI)); + } + + @Test + public void expm1Function() throws IOException { + SearchHit[] hits = query( + "SELECT EXPM1(2) AS expm1" + ); + double expm1 = (double) getField(hits[0], "expm1"); + assertThat(expm1, equalTo(Math.expm1(2))); + } + + @Test + public void degreesFunction() throws IOException { + SearchHit[] hits = query( + "SELECT age, DEGREES(age) AS degrees" + ); + for (SearchHit hit : hits) { + int age = (int) getFieldFromSource(hit, "age"); + double degrees = (double) getField(hit, "degrees"); + assertThat(degrees, equalTo(Math.toDegrees(age))); + } + } + + @Test + public void radiansFunction() throws IOException { + SearchHit[] hits = query( + "SELECT age, RADIANS(age) as radians" + ); + for (SearchHit hit : hits) { + int age = (int) getFieldFromSource(hit, "age"); + double radians = (double) getField(hit, "radians"); + assertThat(radians, equalTo(Math.toRadians(age))); + } + } + + @Test + public void sin() throws IOException { + SearchHit[] hits = query( + "SELECT SIN(PI()) as sin" + ); + double sin = (double) getField(hits[0], "sin"); + assertThat(sin, equalTo(Math.sin(Math.PI))); + } + + @Test + public void asin() throws IOException { + SearchHit[] hits = query( + "SELECT ASIN(PI()) as asin" + ); + double asin = Double.valueOf((String) getField(hits[0], "asin")); + assertThat(asin, equalTo(Math.asin(Math.PI))); + } + + @Test + public void sinh() throws IOException { + SearchHit[] hits = query( + "SELECT SINH(PI()) as sinh" + ); + double sinh = (double) getField(hits[0], "sinh"); + assertThat(sinh, equalTo(Math.sinh(Math.PI))); + } + + @Test + public void power() throws IOException { + SearchHit[] hits = query( + "SELECT POWER(age, 2) AS power", + "WHERE (age IS NOT NULL) AND (balance IS NOT NULL) and (POWER(balance, 3) > 0)" + ); + double power = (double) getField(hits[0], "power"); + assertTrue(power >= 0); + } + + @Test + public void atan2() throws IOException { + SearchHit[] hits = query( + "SELECT ATAN2(age, age) AS atan2", + "WHERE (age IS NOT NULL) AND (ATAN2(age, age) > 0)" + ); + double atan2 = (double) getField(hits[0], "atan2"); + assertThat(atan2, equalTo(Math.atan2(1, 1))); + } + + @Test + public void cot() throws IOException { + SearchHit[] hits = query( + "SELECT COT(PI()) AS cot" + ); + double cot = (double) getField(hits[0], "cot"); + assertThat(cot, closeTo(1 / Math.tan(Math.PI), 0.001)); + } + + @Test + public void sign() throws IOException { + SearchHit[] hits = query( + "SELECT SIGN(E()) AS sign" + ); + double sign = (double) getField(hits[0], "sign"); + assertThat(sign, equalTo(Math.signum(Math.E))); + } + + @Test + public void logWithOneParam() throws IOException { + SearchHit[] hits = query("SELECT LOG(3) AS log"); + double log = (double) getField(hits[0], "log"); + assertThat(log, equalTo(Math.log(3))); + } + + @Test + public void logWithTwoParams() throws IOException { + SearchHit[] hits = query("SELECT LOG(2, 3) AS log"); + double log = (double) getField(hits[0], "log"); + assertThat(log, closeTo(Math.log(3) / Math.log(2), 0.0001)); + } + + @Test + public void logInAggregationShouldPass() { + assertThat( + executeQuery( + "SELECT LOG(age) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + + " WHERE age IS NOT NULL GROUP BY LOG(age) ORDER BY LOG(age)", "jdbc" + ), + containsString("\"type\": \"double\"") + ); + assertThat( + executeQuery( + "SELECT LOG(2, age) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + + " WHERE age IS NOT NULL GROUP BY LOG(2, age) ORDER BY LOG(2, age)", "jdbc" + ), + containsString("\"type\": \"double\"") + ); + } + + @Test + public void log10Test() throws IOException { + SearchHit[] hits = query("SELECT log10(1000) AS log10"); + double log10 = (double) getField(hits[0], "log10"); + assertThat(log10, equalTo(3.0)); + } + + @Test + public void ln() throws IOException { + SearchHit[] hits = query("SELECT LN(5) AS ln"); + double ln = (double) getField(hits[0], "ln"); + assertThat(ln, equalTo(Math.log(5))); + } + + @Test + public void lnInAggregationShouldPass() { + assertThat( + executeQuery( + "SELECT LN(age) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + + " WHERE age IS NOT NULL GROUP BY LN(age) ORDER BY LN(age)", "jdbc" + ), + containsString("\"type\": \"double\"") + ); + } + + @Test + public void rand() throws IOException { + SearchHit[] hits = query("SELECT RAND() AS rand", "ORDER BY rand"); + for (SearchHit hit : hits) { + double rand = (double) getField(hit, "rand"); + assertTrue(rand >= 0 && rand < 1); + } + } + + private SearchHit[] query(String select, String... statements) throws IOException { + final String response = + executeQueryWithStringOutput(select + " " + FROM + " " + String.join(" ", statements)); + + final XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser( + NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, + response); + return SearchResponse.fromXContent(parser).getHits().getHits(); + } + + private Object getField(SearchHit hit, String fieldName) { + return hit.field(fieldName).getValue(); + } + + private Object getFieldFromSource(SearchHit hit, String fieldName) { + return hit.getSourceAsMap().get(fieldName); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MetaDataQueriesIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MetaDataQueriesIT.java new file mode 100644 index 0000000000..d76cd8362a --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MetaDataQueriesIT.java @@ -0,0 +1,436 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_GAME_OF_THRONES; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifySome; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.Matchers.not; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.elasticsearch.client.Request; +import org.hamcrest.Description; +import org.hamcrest.TypeSafeMatcher; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Test; + + +/** + * The following are tests for SHOW/DESCRIBE query support under Pretty Format Response protocol using JDBC format. + *

+ * Unlike SELECT queries, the JDBC format response of SHOW and DESCRIBE queries has determined "schema" fields. + *

+ * Since these integration tests are receiving the JSON response as output, "datarows" values can't be validated by + * key since it is a JSONArray, so the expected length of "schema" will be used instead as well as the expected + * position of the field data in "datarows". + *

+ * These are the outputs of "schema" for SHOW and DESCRIBE, the position of the value in "datarows" will match the + * position of the field in "schema": + *

+ * 1) SHOW query (based on the getTables() method listed here https://docs.oracle.com/javase/8/docs/api/java/sql/DatabaseMetaData.html) + * "schema": [ + * { + * "name": "TABLE_CAT", + * "type": "keyword" + * }, + * { + * "name": "TABLE_SCHEM", + * "type": "keyword" + * }, + * { + * "name": "TABLE_NAME", + * "type": "keyword" + * }, + * { + * "name": "TABLE_TYPE", + * "type": "keyword" + * }, + * { + * "name": "REMARKS", + * "type": "keyword" + * }, + * { + * "name": "TYPE_CAT", + * "type": "keyword" + * }, + * { + * "name": "TYPE_SCHEM", + * "type": "keyword" + * }, + * { + * "name": "TYPE_NAME", + * "type": "keyword" + * }, + * { + * "name": "SELF_REFERENCING_COL_NAME", + * "type": "keyword" + * }, + * { + * "name": "REF_GENERATION", + * "type": "keyword" + * } + * ] + *

+ * 2) DESCRIBE query (based on the getColumns() method listed here https://docs.oracle.com/javase/8/docs/api/java/sql/DatabaseMetaData.html) + * "schema": [ + * { + * "name": "TABLE_CAT", + * "type": "keyword" + * }, + * { + * "name": "TABLE_SCHEM", + * "type": "keyword" + * }, + * { + * "name": "TABLE_NAME", + * "type": "keyword" + * }, + * { + * "name": "COLUMN_NAME", + * "type": "keyword" + * }, + * { + * "name": "DATA_TYPE", + * "type": "integer" + * }, + * { + * "name": "TYPE_NAME", + * "type": "keyword" + * }, + * { + * "name": "COLUMN_SIZE", + * "type": "integer" + * }, + * { + * "name": "BUFFER_LENGTH", + * "type": "integer" + * }, + * { + * "name": "DECIMAL_DIGITS", + * "type": "integer" + * }, + * { + * "name": "NUM_PREC_RADIX", + * "type": "integer" + * }, + * { + * "name": "NULLABLE", + * "type": "integer" + * }, + * { + * "name": "REMARKS", + * "type": "keyword" + * }, + * { + * "name": "COLUMN_DEF", + * "type": "keyword" + * }, + * { + * "name": "SQL_DATA_TYPE", + * "type": "integer" + * }, + * { + * "name": "SQL_DATETIME_SUB", + * "type": "integer" + * }, + * { + * "name": "CHAR_OCTET_LENGTH", + * "type": "integer" + * }, + * { + * "name": "ORDINAL_POSITION", + * "type": "integer" + * }, + * { + * "name": "IS_NULLABLE", + * "type": "keyword" + * }, + * { + * "name": "SCOPE_CATALOG", + * "type": "keyword" + * }, + * { + * "name": "SCOPE_SCHEMA", + * "type": "keyword" + * }, + * { + * "name": "SCOPE_TABLE", + * "type": "keyword" + * }, + * { + * "name": "SOURCE_DATA_TYPE", + * "type": "short" + * }, + * { + * "name": "IS_AUTOINCREMENT", + * "type": "keyword" + * }, + * { + * "name": "IS_GENERATEDCOLUMN", + * "type": "keyword" + * } + * ] + */ +public class MetaDataQueriesIT extends SQLIntegTestCase { + + // Number of fields in the response, assuming no filters, based on output shown above + private static final int SHOW_FIELD_LENGTH = 10; + + private static final int DESCRIBE_FIELD_LENGTH = 24; + + private static final String TABLE_TYPE = "BASE TABLE"; + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + loadIndex(Index.PHRASE); + loadIndex(Index.GAME_OF_THRONES); + } + + @Override + protected Request getSqlRequest(String request, boolean explain) { + Request sqlRequest = super.getSqlRequest(request, explain); + sqlRequest.addParameter("format", "jdbc"); + + return sqlRequest; + } + + @Test + public void showSingleIndex() throws IOException { + JSONObject response = + executeQuery(String.format("SHOW TABLES LIKE %s", TestsConstants.TEST_INDEX_ACCOUNT)); + + String[] fields = {"TABLE_CAT", "TABLE_NAME", "TABLE_TYPE"}; + checkContainsColumns(getSchema(response), fields); + + JSONArray dataRows = getDataRows(response); + assertThat(dataRows.length(), equalTo(1)); + assertThat(dataRows.getJSONArray(0).length(), equalTo(SHOW_FIELD_LENGTH)); + + /* + * Assumed indices of fields in dataRows based on "schema" output for SHOW given above: + * "TABLE_CAT" : 0 + * "TABLE_NAME" : 2 + * "TABLE_TYPE" : 3 + */ + JSONArray row = dataRows.getJSONArray(0); + assertThat(row.get(0), equalTo(getClusterName())); + assertThat(row.get(2), equalTo(TestsConstants.TEST_INDEX_ACCOUNT)); + assertThat(row.get(3), equalTo(TABLE_TYPE)); + } + + @Test + public void showCaseSensitivityCheck() throws IOException { + JSONObject response = + executeQuery(String.format("show tables like %s", TestsConstants.TEST_INDEX_ACCOUNT)); + + String[] fields = {"TABLE_CAT", "TABLE_NAME", "TABLE_TYPE"}; + checkContainsColumns(getSchema(response), fields); + + JSONArray dataRows = getDataRows(response); + assertThat(dataRows.length(), equalTo(1)); + assertThat(dataRows.getJSONArray(0).length(), equalTo(SHOW_FIELD_LENGTH)); + + JSONArray row = dataRows.getJSONArray(0); + assertThat(row.get(0), equalTo(getClusterName())); + assertThat(row.get(2), equalTo(TestsConstants.TEST_INDEX_ACCOUNT)); + assertThat(row.get(3), equalTo(TABLE_TYPE)); + } + + @Test + public void showWildcardIndex() throws IOException { + JSONObject response = + executeQuery(String.format("SHOW TABLES LIKE %s%%", TestsConstants.TEST_INDEX)); + + String pattern = String.format("%s.*", TestsConstants.TEST_INDEX); + JSONArray dataRows = getDataRows(response); + assertThat(dataRows.length(), equalTo(3)); + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + String tableName = row.getString(2); + + assertTrue(tableName.matches(pattern)); + } + } + + @Test + public void describeSingleIndex() throws IOException { + JSONObject response = + executeQuery(String.format("DESCRIBE TABLES LIKE %s", TestsConstants.TEST_INDEX_ACCOUNT)); + + // Schema for DESCRIBE is filled with a lot of fields that aren't used so only the important + // ones are checked for here + String[] fields = {"TABLE_NAME", "COLUMN_NAME", "TYPE_NAME"}; + checkContainsColumns(getSchema(response), fields); + + JSONArray dataRows = getDataRows(response); + assertThat(dataRows.length(), greaterThan(0)); + assertThat(dataRows.getJSONArray(0).length(), equalTo(DESCRIBE_FIELD_LENGTH)); + + /* + * Assumed indices of fields in dataRows based on "schema" output for DESCRIBE given above: + * "TABLE_NAME" : 2 + * "COLUMN_NAME" : 3 + * "TYPE_NAME" : 5 + */ + JSONArray row = dataRows.getJSONArray(0); + assertThat(row.get(2), equalTo(TestsConstants.TEST_INDEX_ACCOUNT)); + assertThat(row.get(3), not(equalTo(JSONObject.NULL))); + assertThat(row.get(5), not(equalTo(JSONObject.NULL))); + } + + @Test + public void describeSingleIndexWithObjectFieldShouldPass() throws IOException { + JSONObject response = + executeQuery(String.format("DESCRIBE TABLES LIKE %s", TEST_INDEX_GAME_OF_THRONES)); + + // Schema for DESCRIBE is filled with a lot of fields that aren't used so only the important + // ones are checked for here + String[] fields = {"TABLE_NAME", "COLUMN_NAME", "TYPE_NAME"}; + checkContainsColumns(getSchema(response), fields); + + JSONArray dataRows = getDataRows(response); + assertThat(dataRows.length(), greaterThan(0)); + assertThat(dataRows.getJSONArray(0).length(), equalTo(DESCRIBE_FIELD_LENGTH)); + + verifySome(dataRows, + describeRow(TEST_INDEX_GAME_OF_THRONES, "nickname", "text"), + describeRow(TEST_INDEX_GAME_OF_THRONES, "name", "object"), + describeRow(TEST_INDEX_GAME_OF_THRONES, "name.firstname", "text"), + describeRow(TEST_INDEX_GAME_OF_THRONES, "name.lastname", "text"), + describeRow(TEST_INDEX_GAME_OF_THRONES, "name.ofHerName", "integer"), + describeRow(TEST_INDEX_GAME_OF_THRONES, "name.ofHisName", "integer"), + describeRow(TEST_INDEX_GAME_OF_THRONES, "house", "text"), + describeRow(TEST_INDEX_GAME_OF_THRONES, "gender", "text")); + } + + @Test + public void describeCaseSensitivityCheck() throws IOException { + JSONObject response = + executeQuery(String.format("describe tables like %s", TestsConstants.TEST_INDEX_ACCOUNT)); + + String[] fields = {"TABLE_NAME", "COLUMN_NAME", "TYPE_NAME"}; + checkContainsColumns(getSchema(response), fields); + + JSONArray dataRows = getDataRows(response); + assertThat(dataRows.length(), greaterThan(0)); + assertThat(dataRows.getJSONArray(0).length(), equalTo(DESCRIBE_FIELD_LENGTH)); + + JSONArray row = dataRows.getJSONArray(0); + assertThat(row.get(2), equalTo(TestsConstants.TEST_INDEX_ACCOUNT)); + assertThat(row.get(3), not(equalTo(JSONObject.NULL))); + assertThat(row.get(5), not(equalTo(JSONObject.NULL))); + } + + @Test + public void describeWildcardIndex() throws IOException { + JSONObject response = + executeQuery(String.format("DESCRIBE TABLES LIKE %s%%", TestsConstants.TEST_INDEX)); + + String pattern = String.format("%s.*", TestsConstants.TEST_INDEX); + JSONArray dataRows = getDataRows(response); + assertThat(dataRows.length(), greaterThan(0)); + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + String tableName = row.getString(2); + + assertTrue(tableName.matches(pattern)); + } + } + + @Test + public void describeWildcardColumn() throws IOException { + JSONObject response = executeQuery(String.format("DESCRIBE TABLES LIKE %s COLUMNS LIKE %%name", + TestsConstants.TEST_INDEX_ACCOUNT)); + + String pattern = ".*name"; + JSONArray dataRows = getDataRows(response); + assertThat(dataRows.length(), greaterThan(0)); + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + String columnName = row.getString(3); + + assertTrue(columnName.matches(pattern)); + } + } + + @Test + public void describeSingleCharacterWildcard() throws IOException { + JSONObject response = executeQuery(String.format("DESCRIBE TABLES LIKE %s COLUMNS LIKE %%na_e", + TestsConstants.TEST_INDEX_ACCOUNT)); + + String pattern = ".*na.e"; + JSONArray dataRows = getDataRows(response); + assertThat(dataRows.length(), greaterThan(0)); + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + String columnName = row.getString(3); + + assertTrue(columnName.matches(pattern)); + } + } + + private JSONArray getSchema(JSONObject jdbcResponse) { + return jdbcResponse.getJSONArray("schema"); + } + + private JSONArray getDataRows(JSONObject jdbcResponse) { + return jdbcResponse.getJSONArray("datarows"); + } + + private void checkContainsColumns(JSONArray schema, String... fields) { + List columnNames = new ArrayList<>(); + for (int i = 0; i < schema.length(); i++) { + JSONObject column = schema.getJSONObject(i); + columnNames.add(column.getString("name")); + } + + assertThat(columnNames, hasItems(fields)); + } + + private String getClusterName() throws IOException { + String response = executeRequest(new Request("GET", "_cluster/health")); + return new JSONObject(response).optString("cluster_name", ""); + } + + public static TypeSafeMatcher describeRow(Object... expectedObjects) { + return new TypeSafeMatcher() { + @Override + public void describeTo(Description description) { + description.appendText(String.join(",", Arrays.asList(expectedObjects).toString())); + } + + @Override + protected boolean matchesSafely(JSONArray array) { + List actualObjects = new ArrayList<>(); + // TABLE_NAME + actualObjects.add(array.get(2)); + // COLUMN_NAME + actualObjects.add(array.get(3)); + // TYPE_NAME + actualObjects.add(array.get(5)); + return Arrays.asList(expectedObjects).equals(actualObjects); + } + }; + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MethodQueryIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MethodQueryIT.java new file mode 100644 index 0000000000..6cb584d85c --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MethodQueryIT.java @@ -0,0 +1,127 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static org.hamcrest.Matchers.both; +import static org.hamcrest.Matchers.containsString; + +import java.io.IOException; +import java.util.Locale; +import org.junit.Assert; +import org.junit.Test; + +/** + * 定製方法查詢． + * + * @author ansj + */ +public class MethodQueryIT extends SQLIntegTestCase { + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + } + + /** + * query 搜索就是　，　lucene 原生的搜素方式 注意这个例子中ｖａｌｕｅ可以随便命名 "query" : + * {query_string" : {"query" : "address:880 Holmes Lane"} + * + * @throws IOException + */ + @Test + public void queryTest() throws IOException { + final String result = explainQuery(String.format(Locale.ROOT, + "select address from %s where query('address:880 Holmes Lane') limit 3", + TestsConstants.TEST_INDEX_ACCOUNT)); + Assert.assertThat(result, + containsString("query_string\":{\"query\":\"address:880 Holmes Lane")); + + } + + /** + * matchQuery 是利用分词结果进行单个字段的搜索． "query" : { "match" : { "address" : + * {"query":"880 Holmes Lane", "type" : "boolean" } } } + * + * @throws IOException + */ + @Test + public void matchQueryTest() throws IOException { + final String result = explainQuery(String.format(Locale.ROOT, + "select address from %s where address= matchQuery('880 Holmes Lane') limit 3", + TestsConstants.TEST_INDEX_ACCOUNT)); + Assert.assertThat(result, + containsString("{\"match\":{\"address\":{\"query\":\"880 Holmes Lane\"")); + } + + /** + * matchQuery 是利用分词结果进行单个字段的搜索． "query" : { "bool" : { "must" : { "bool" : { + * "should" : [ { "constant_score" : { "query" : { "match" : { "address" : { + * "query" : "Lane", "type" : "boolean" } } }, "boost" : 100.0 } }, { + * "constant_score" : { "query" : { "match" : { "address" : { "query" : + * "Street", "type" : "boolean" } } }, "boost" : 0.5 } } ] } } } } + * + * @throws IOException + */ + // todo + @Test + public void scoreQueryTest() throws IOException { + final String result = explainQuery(String.format(Locale.ROOT, + "select address from %s " + + "where score(matchQuery(address, 'Lane'),100) " + + "or score(matchQuery(address,'Street'),0.5) order by _score desc limit 3", + TestsConstants.TEST_INDEX_ACCOUNT)); + Assert.assertThat(result, + both(containsString("{\"constant_score\":" + + "{\"filter\":{\"match\":{\"address\":{\"query\":\"Lane\"")).and( + containsString("{\"constant_score\":" + + "{\"filter\":{\"match\":{\"address\":{\"query\":\"Street\""))); + } + + /** + * wildcardQuery 是用通配符的方式查找某个term 　比如例子中 l*e means leae ltae .... + * "wildcard": { "address" : { "wildcard" : "l*e" } } + * + * @throws IOException + */ + @Test + public void wildcardQueryTest() throws IOException { + final String result = explainQuery(String.format(Locale.ROOT, + "select address from %s where address= wildcardQuery('l*e') order by _score desc limit 3", + TestsConstants.TEST_INDEX_ACCOUNT)); + Assert.assertThat(result, + containsString("{\"wildcard\":{\"address\":{\"wildcard\":\"l*e\"")); + } + + /** + * matchPhraseQueryTest 短语查询完全匹配． + * "address" : { + * "query" : "671 Bristol Street", + * "type" : "phrase" + * } + * + * @throws IOException + */ + @Test + public void matchPhraseQueryTest() throws IOException { + final String result = explainQuery(String.format(Locale.ROOT, + "select address from %s " + + "where address= matchPhrase('671 Bristol Street') order by _score desc limit 3", + TestsConstants.TEST_INDEX_ACCOUNT)); + Assert.assertThat(result, + containsString("{\"match_phrase\":{\"address\":{\"query\":\"671 Bristol Street\"")); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MetricsIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MetricsIT.java new file mode 100644 index 0000000000..ef233dc6c6 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MetricsIT.java @@ -0,0 +1,82 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_DOG; +import static org.hamcrest.Matchers.equalTo; + +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.MetricName; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.concurrent.TimeUnit; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.json.JSONObject; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + +@Ignore +public class MetricsIT extends SQLIntegTestCase { + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + loadIndex(Index.DOG); + } + + @Test + public void requestCount() throws IOException, InterruptedException { + multiQueries(3); + TimeUnit.SECONDS.sleep(2L); + JSONObject jsonObject = new JSONObject(executeStatRequest(makeStatRequest())); + assertThat(jsonObject.getInt(MetricName.REQ_COUNT_TOTAL.getName()), equalTo(3)); + } + + private void multiQueries(int n) throws IOException { + for (int i = 0; i < n; ++i) { + executeQuery(String.format("SELECT COUNT(*) FROM %s/dog", TEST_INDEX_DOG)); + } + } + + private Request makeStatRequest() { + return new Request( + "GET", "/_opendistro/_sql/stats" + ); + } + + private String executeStatRequest(final Request request) throws IOException { + + Response sqlResponse = client().performRequest(request); + + Assert.assertTrue(sqlResponse.getStatusLine().getStatusCode() == 200); + + InputStream is = sqlResponse.getEntity().getContent(); + StringBuilder sb = new StringBuilder(); + try (BufferedReader br = new BufferedReader(new InputStreamReader(is))) { + String line = null; + while ((line = br.readLine()) != null) { + sb.append(line); + } + } + + return sb.toString(); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MultiQueryIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MultiQueryIT.java new file mode 100644 index 0000000000..6360408832 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/MultiQueryIT.java @@ -0,0 +1,324 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItems; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Assert; +import org.junit.Test; + +public class MultiQueryIT extends SQLIntegTestCase { + + private static String MINUS_SCROLL_DEFAULT_HINT = + " /*! MINUS_SCROLL_FETCH_AND_RESULT_LIMITS(1000, 50, 100) */ "; + private static String MINUS_TERMS_OPTIMIZATION_HINT = + " /*! MINUS_USE_TERMS_OPTIMIZATION(true) */ "; + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + loadIndex(Index.DOG); + loadIndex(Index.GAME_OF_THRONES); + loadIndex(Index.SYSTEM); + } + + @Test + public void unionAllSameRequestOnlyOneRecordTwice() throws IOException { + String query = String.format("SELECT firstname " + + "FROM %s " + + "WHERE firstname = 'Amber' " + + "LIMIT 1 " + + "UNION ALL " + + "SELECT firstname " + + "FROM %s " + + "WHERE firstname = 'Amber'", + TestsConstants.TEST_INDEX_ACCOUNT, TestsConstants.TEST_INDEX_ACCOUNT); + + JSONObject response = executeQuery(query); + JSONArray hits = getHits(response); + + assertThat(getHits(response).length(), equalTo(2)); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + JSONObject source = getSource(hit); + + assertThat(source.getString("firstname"), equalTo("Amber")); + } + } + + @Test + public void unionAllOnlyOneRecordEachWithAlias() throws IOException { + String query = String.format("SELECT firstname FROM %s WHERE firstname = 'Amber' " + + "UNION ALL " + + "SELECT dog_name as firstname FROM %s WHERE dog_name = 'rex'", + TestsConstants.TEST_INDEX_ACCOUNT, TestsConstants.TEST_INDEX_DOG); + + JSONObject response = executeQuery(query); + assertThat(getHits(response).length(), equalTo(2)); + + Set names = new HashSet<>(); + JSONArray hits = getHits(response); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + JSONObject source = getSource(hit); + + names.add(source.getString("firstname")); + } + + assertThat(names, hasItems("Amber", "rex")); + } + + @Test + public void unionAllOnlyOneRecordEachWithComplexAlias() throws IOException { + String query = String.format("SELECT firstname FROM %s WHERE firstname = 'Amber' " + + "UNION ALL " + + "SELECT name.firstname as firstname " + + "FROM %s " + + "WHERE name.firstname = 'daenerys'", + TestsConstants.TEST_INDEX_ACCOUNT, TestsConstants.TEST_INDEX_GAME_OF_THRONES); + + JSONObject response = executeQuery(query); + assertThat(getHits(response).length(), equalTo(2)); + + Set names = new HashSet<>(); + JSONArray hits = getHits(response); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + JSONObject source = getSource(hit); + + names.add(source.getString("firstname")); + } + + assertThat(names, hasItems("Amber", "Daenerys")); + } + + @Test + public void minusAMinusANoAlias() throws IOException { + innerMinusAMinusANoAlias(""); + } + + @Test + public void minusAMinusANoAliasWithScrolling() throws IOException { + innerMinusAMinusANoAlias(MINUS_SCROLL_DEFAULT_HINT); + } + + @Test + public void minusAMinusANoAliasWithScrollingAndTerms() throws IOException { + innerMinusAMinusANoAlias(MINUS_SCROLL_DEFAULT_HINT + MINUS_TERMS_OPTIMIZATION_HINT); + } + + @Test + public void minusAMinusBNoAlias() throws IOException { + innerMinusAMinusBNoAlias(""); + } + + @Test + public void minusAMinusBNoAliasWithScrolling() throws IOException { + innerMinusAMinusBNoAlias(MINUS_SCROLL_DEFAULT_HINT); + } + + @Test + public void minusAMinusBNoAliasWithScrollingAndTerms() throws IOException { + innerMinusAMinusBNoAlias(MINUS_SCROLL_DEFAULT_HINT + MINUS_TERMS_OPTIMIZATION_HINT); + } + + @Test + public void minusCMinusDTwoFieldsNoAlias() throws IOException { + innerMinusCMinusDTwoFieldsNoAlias(""); + } + + @Test + public void minusCMinusDTwoFieldsNoAliasWithScrolling() throws IOException { + innerMinusCMinusDTwoFieldsNoAlias(MINUS_SCROLL_DEFAULT_HINT); + } + + @Test + public void minusCMinusDTwoFieldsAliasOnBothSecondTableFields() throws IOException { + String query = String.format("SELECT pk, letter FROM %s WHERE system_name = 'C' " + + "MINUS " + + "SELECT myId as pk, myLetter as letter FROM %s WHERE system_name = 'E'", + TestsConstants.TEST_INDEX_SYSTEM, TestsConstants.TEST_INDEX_SYSTEM); + + JSONObject response = executeQuery(query); + assertThat(getHits(response).length(), equalTo(1)); + + JSONObject hit = getHits(response).getJSONObject(0); + JSONObject source = getSource(hit); + + assertThat(source.length(), equalTo(2)); + Assert.assertTrue("Source expected to contain 'pk' field", source.has("pk")); + Assert.assertTrue("Source expected to contain 'letter' field", source.has("letter")); + assertThat(source.getInt("pk"), equalTo(1)); + assertThat(source.getString("letter"), equalTo("e")); + } + + @Test + public void minusCMinusDTwoFieldsAliasOnBothTables() throws IOException { + innerMinusCMinusDTwoFieldsAliasOnBothTables(""); + } + + @Test + public void minusCMinusDTwoFieldsAliasOnBothTablesWithScrolling() throws IOException { + innerMinusCMinusDTwoFieldsAliasOnBothTables(MINUS_SCROLL_DEFAULT_HINT); + } + + @Test + public void minusCMinusCTwoFieldsOneAlias() throws IOException { + String query = String.format("SELECT pk as myId, letter FROM %s WHERE system_name = 'C' " + + "MINUS " + + "SELECT pk as myId, letter FROM %s WHERE system_name = 'C'", + TestsConstants.TEST_INDEX_SYSTEM, TestsConstants.TEST_INDEX_SYSTEM); + + JSONObject response = executeQuery(query); + assertThat(getHits(response).length(), equalTo(0)); + } + + @Test + public void minusCMinusTNonExistentFieldTwoFields() throws IOException { + String query = String.format("SELECT pk, letter FROM %s WHERE system_name = 'C' " + + "MINUS " + + "SELECT pk, letter FROM %s WHERE system_name = 'T' ", + TestsConstants.TEST_INDEX_SYSTEM, TestsConstants.TEST_INDEX_SYSTEM); + + JSONObject response = executeQuery(query); + assertThat(getHits(response).length(), equalTo(3)); + } + + @Test + public void minusCMinusTNonExistentFieldOneField() throws IOException { + innerMinusCMinusTNonExistentFieldOneField(""); + } + + @Test + public void minusCMinusTNonExistentOneFieldWithScrolling() throws IOException { + innerMinusCMinusTNonExistentFieldOneField(MINUS_SCROLL_DEFAULT_HINT); + } + + @Test + public void minusCMinusTNonExistentFieldOneFieldWithScrollingAndOptimization() + throws IOException { + innerMinusCMinusTNonExistentFieldOneField( + MINUS_SCROLL_DEFAULT_HINT + MINUS_TERMS_OPTIMIZATION_HINT); + } + + @Test + public void minusTMinusCNonExistentFieldFirstQuery() throws IOException { + innerMinusTMinusCNonExistentFieldFirstQuery(""); + } + + @Test + public void minusTMinusCNonExistentFieldFirstQueryWithScrolling() throws IOException { + innerMinusTMinusCNonExistentFieldFirstQuery(MINUS_SCROLL_DEFAULT_HINT); + } + + @Test + public void minusTMinusCNonExistentFieldFirstQueryWithScrollingAndOptimization() + throws IOException { + innerMinusTMinusCNonExistentFieldFirstQuery( + MINUS_SCROLL_DEFAULT_HINT + MINUS_TERMS_OPTIMIZATION_HINT); + } + + private void innerMinusAMinusANoAlias(String hint) throws IOException { + String query = String.format("SELECT %s pk FROM %s WHERE system_name = 'A' " + + "MINUS " + + "SELECT pk FROM %s WHERE system_name = 'A'", + hint, TestsConstants.TEST_INDEX_SYSTEM, TestsConstants.TEST_INDEX_SYSTEM); + + JSONObject response = executeQuery(query); + assertThat(getHits(response).length(), equalTo(0)); + } + + private void innerMinusAMinusBNoAlias(String hint) throws IOException { + String query = String.format("SELECT %s pk FROM %s WHERE system_name = 'A' " + + "MINUS " + + "SELECT pk FROM %s WHERE system_name = 'B'", + hint, TestsConstants.TEST_INDEX_SYSTEM, TestsConstants.TEST_INDEX_SYSTEM); + + JSONObject response = executeQuery(query); + assertThat(getHits(response).length(), equalTo(1)); + + JSONObject hit = getHits(response).getJSONObject(0); + JSONObject source = getSource(hit); + assertThat(source.length(), equalTo(1)); + Assert.assertTrue("Source expected to contain 'pk' field", source.has("pk")); + assertThat(source.getInt("pk"), equalTo(3)); + } + + private void innerMinusCMinusDTwoFieldsNoAlias(String hint) throws IOException { + String query = String.format("SELECT %s pk, letter FROM %s WHERE system_name = 'C' " + + "MINUS " + + "SELECT pk, letter FROM %s WHERE system_name = 'D'", + hint, TestsConstants.TEST_INDEX_SYSTEM, TestsConstants.TEST_INDEX_SYSTEM); + + JSONObject response = executeQuery(query); + assertThat(getHits(response).length(), equalTo(1)); + + JSONObject hit = getHits(response).getJSONObject(0); + JSONObject source = getSource(hit); + + assertThat(source.length(), equalTo(2)); + Assert.assertTrue("Source expected to contain 'pk' field", source.has("pk")); + Assert.assertTrue("Source expected to contain 'letter' field", source.has("letter")); + assertThat(source.getInt("pk"), equalTo(1)); + assertThat(source.getString("letter"), equalTo("e")); + } + + private void innerMinusCMinusDTwoFieldsAliasOnBothTables(String hint) throws IOException { + String query = String.format("SELECT %s pk as myId, letter FROM %s WHERE system_name = 'C' " + + "MINUS " + + "SELECT myId, myLetter as letter FROM %s WHERE system_name = 'E'", + hint, TestsConstants.TEST_INDEX_SYSTEM, TestsConstants.TEST_INDEX_SYSTEM); + + JSONObject response = executeQuery(query); + assertThat(getHits(response).length(), equalTo(1)); + + JSONObject hit = getHits(response).getJSONObject(0); + JSONObject source = getSource(hit); + + assertThat(source.length(), equalTo(2)); + Assert.assertTrue("Source expected to contain 'pk' field as 'myId'", source.has("myId")); + Assert.assertTrue("Source expected to contain 'letter' field", source.has("letter")); + assertThat(source.getInt("myId"), equalTo(1)); + assertThat(source.getString("letter"), equalTo("e")); + } + + private void innerMinusCMinusTNonExistentFieldOneField(String hint) throws IOException { + String query = String.format("SELECT %s letter FROM %s WHERE system_name = 'C' " + + "MINUS " + + "SELECT letter FROM %s WHERE system_name = 'T'", + hint, TestsConstants.TEST_INDEX_SYSTEM, TestsConstants.TEST_INDEX_SYSTEM); + + JSONObject response = executeQuery(query); + assertThat(getHits(response).length(), equalTo(3)); + } + + private void innerMinusTMinusCNonExistentFieldFirstQuery(String hint) throws IOException { + String query = String.format("SELECT %s letter FROM %s WHERE system_name = 'T' " + + "MINUS " + + "SELECT letter FROM %s WHERE system_name = 'C'", + hint, TestsConstants.TEST_INDEX_SYSTEM, TestsConstants.TEST_INDEX_SYSTEM); + + JSONObject response = executeQuery(query); + assertThat(getHits(response).length(), equalTo(0)); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/NestedFieldQueryIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/NestedFieldQueryIT.java new file mode 100644 index 0000000000..be50f28db3 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/NestedFieldQueryIT.java @@ -0,0 +1,917 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.hitAll; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.kvString; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.function.Function; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.SearchHit; +import org.hamcrest.BaseMatcher; +import org.hamcrest.Description; +import org.hamcrest.FeatureMatcher; +import org.hamcrest.Matcher; +import org.hamcrest.core.Is; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + +/** + * Integration test cases for both rewriting and projection logic. + *

+ * Test result: + * 1) SELECT * or any field or aggregate function on any field + * 2) WHERE single or multiple conditions on nested type + * 3) GROUP BY regular field and aggregate on any field + * 4) GROUP BY nested field and COUNT(*) + * 5) UNION/MINUS but only SELECT nested field + *

+ * Does NOT support: + * 1) GROUP BY nested field and aggregate other than COUNT + * 2) UNION/MINUS and SELECT nested field (need to flatten during set computation) + * 3) JOIN (doesn't work if put alias before nested field name and may have similar problem as UNION/MINUS during computation) + * 4) Subquery + * 5) HAVING + * 6) Verification for conditions mixed with regular and nested fields + */ +public class NestedFieldQueryIT extends SQLIntegTestCase { + + private static final String FROM = + "FROM " + TestsConstants.TEST_INDEX_NESTED_TYPE + " n, n.message m"; + + + @Override + protected void init() throws Exception { + loadIndex(Index.NESTED); + loadIndex(Index.EMPLOYEE_NESTED); + } + + @Test + public void selectAll() throws IOException { + queryAll("SELECT *"); + } + + @Test + public void noCondition() throws IOException { + queryAll("SELECT myNum, someField, m.author, m.info"); + } + + private void queryAll(String sql) throws IOException { + assertThat( + query(sql), + hits( + hit( + myNum(1), + someField("b"), + innerHits("message", + hit( + author("e"), + info("a") + ) + ) + ), + hit( + myNum(2), + someField("a"), + innerHits("message", + hit( + author("f"), + info("b") + ) + ) + ), + hit( + myNum(3), + someField("a"), + innerHits("message", + hit( + author("g"), + info("c") + ) + ) + ), + hit( + myNum(4), + someField("b"), + innerHits("message", + hit( + author("h"), + info("c") + ), + hit( + author("i"), + info("a") + ) + ) + ), + hit( + myNum(new int[] {3, 4}), + someField("a"), + innerHits("message", + hit( + author("zz"), + info("zz") + ) + ) + ) + ) + ); + } + + @Test + public void singleCondition() throws IOException { + assertThat( + query( + "SELECT myNum, m.author, m.info", + "WHERE m.info = 'c'" + ), + hits( + hit( + myNum(3), + innerHits("message", + hit( + author("g"), + info("c") + ) + ) + ), + hit( + myNum(4), + innerHits("message", + hit( + author("h"), + info("c") + ) + ) + ) + ) + ); + } + + @Test + public void multipleConditionsOfNestedField() throws IOException { + assertThat( + query( + "SELECT someField, m.author, m.info", + "WHERE m.info = 'c' AND m.author = 'h'" + ), + hits( + hit( + someField("b"), + innerHits("message", + hit( + author("h"), + info("c") + ) + ) + ) + ) + ); + } + + @Test + public void multipleConditionsOfNestedFieldNoMatch() throws IOException { + assertThat( + query( + "SELECT someField, m.author, m.info", + "WHERE m.info = 'c' AND m.author = 'i'" + ), + hits() + ); + } + + @Test + public void multipleConditionsOfRegularAndNestedField() throws IOException { + assertThat( + query( + "SELECT myNum, m.author, m.info", + "WHERE myNum = 3 AND m.info = 'c'" + ), + hits( + hit( + myNum(3), + innerHits("message", + hit( + author("g"), + info("c") + ) + ) + ) + ) + ); + } + + @Test + public void multipleConditionsOfRegularOrNestedField() throws IOException { + assertThat( + query( + "SELECT myNum, m.author, m.info", + "WHERE myNum = 2 OR m.info = 'c'" + ), + hits( + hit( + myNum(2) + ), // Note: no inner hit here because of no match in nested field + hit( + myNum(3), + innerHits("message", + hit( + author("g"), + info("c") + ) + ) + ), + hit( + myNum(4), + innerHits("message", + hit( + author("h"), + info("c") + ) + ) + ) + ) + ); + } + + @Test + public void leftJoinSelectAll() throws IOException { + String sql = "SELECT * " + + "FROM elasticsearch-sql_test_index_employee_nested e " + + "LEFT JOIN e.projects p"; + String explain = explainQuery(sql); + assertThat(explain, containsString("{\"bool\":{\"must_not\":[{\"nested\":{\"query\":" + + "{\"exists\":{\"field\":\"projects\",\"boost\":1.0}},\"path\":\"projects\"")); + + assertThat(explain, containsString("\"_source\":{\"includes\":[\"projects.*\"")); + + JSONObject results = executeQuery(sql); + Assert.assertThat(getTotalHits(results), equalTo(4)); + } + + @Test + public void leftJoinSpecificFields() throws IOException { + String sql = "SELECT e.name, p.name, p.started_year " + + "FROM elasticsearch-sql_test_index_employee_nested e " + + "LEFT JOIN e.projects p"; + String explain = explainQuery(sql); + assertThat(explain, containsString("{\"bool\":{\"must_not\":[{\"nested\":{\"query\":" + + "{\"exists\":{\"field\":\"projects\",\"boost\":1.0}},\"path\":\"projects\"")); + assertThat(explain, containsString("\"_source\":{\"includes\":[\"name\"],")); + assertThat(explain, + containsString("\"_source\":{\"includes\":[\"projects.name\",\"projects.started_year\"]")); + + JSONObject results = executeQuery(sql); + Assert.assertThat(getTotalHits(results), equalTo(4)); + } + + @Ignore("Comma join in left join won't pass syntax check in new ANTLR parser. " + + "Ignore for now and require to change grammar too when we want to support this case.") + @Test + public void leftJoinExceptionOnExtraNestedFields() throws IOException { + String sql = "SELECT * " + + "FROM elasticsearch-sql_test_index_employee_nested e " + + "LEFT JOIN e.projects p, e.comments c"; + + try { + String explain = explainQuery(sql); + Assert.fail("Expected ResponseException, but none was thrown"); + } catch (ResponseException e) { + assertThat(e.getResponse().getStatusLine().getStatusCode(), + equalTo(RestStatus.BAD_REQUEST.getStatus())); + final String entity = TestUtils.getResponseBody(e.getResponse()); + assertThat(entity, + containsString("only single nested field is allowed as right table for LEFT JOIN")); + assertThat(entity, containsString("\"type\":\"verification_exception\"")); + } + } + + + @Test + public void aggregationWithoutGroupBy() throws IOException { + String sql = "SELECT AVG(m.dayOfWeek) AS avgDay " + FROM; + + JSONObject result = executeQuery(sql); + JSONObject aggregation = getAggregation(result, "message.dayOfWeek@NESTED"); + + Assert.assertThat((Double) aggregation.query("/avgDay/value"), closeTo(3.166666666, 0.01)); + } + + @Test + public void groupByNestedFieldAndCount() throws IOException { + final String sql = "SELECT m.info, COUNT(*) " + FROM + " GROUP BY m.info"; + + JSONObject result = executeQuery(sql); + JSONObject aggregation = getAggregation(result, "message.info@NESTED"); + JSONArray msgInfoBuckets = (JSONArray) aggregation.optQuery("/message.info/buckets"); + + Assert.assertNotNull(msgInfoBuckets); + Assert.assertThat(msgInfoBuckets.length(), equalTo(4)); + Assert.assertThat(msgInfoBuckets.query("/0/key"), equalTo("a")); + Assert.assertThat(msgInfoBuckets.query("/0/COUNT(*)/value"), equalTo(2)); + Assert.assertThat(msgInfoBuckets.query("/1/key"), equalTo("c")); + Assert.assertThat(msgInfoBuckets.query("/1/COUNT(*)/value"), equalTo(2)); + Assert.assertThat(msgInfoBuckets.query("/2/key"), equalTo("b")); + Assert.assertThat(msgInfoBuckets.query("/2/COUNT(*)/value"), equalTo(1)); + Assert.assertThat(msgInfoBuckets.query("/3/key"), equalTo("zz")); + Assert.assertThat(msgInfoBuckets.query("/3/COUNT(*)/value"), equalTo(1)); + } + + @Test + public void groupByRegularFieldAndSum() throws IOException { + final String sql = "SELECT *, SUM(m.dayOfWeek) AS sumDay " + FROM + " GROUP BY someField"; + + JSONObject result = executeQuery(sql); + JSONObject aggregation = getAggregation(result, "someField"); + JSONArray msgInfoBuckets = (JSONArray) aggregation.optQuery("/buckets"); + + Assert.assertNotNull(msgInfoBuckets); + Assert.assertThat(msgInfoBuckets.length(), equalTo(2)); + Assert.assertThat(msgInfoBuckets.query("/0/key"), equalTo("a")); + Assert.assertThat((Double) msgInfoBuckets.query("/0/message.dayOfWeek@NESTED/sumDay/value"), + closeTo(9.0, 0.01)); + Assert.assertThat(msgInfoBuckets.query("/1/key"), equalTo("b")); + Assert.assertThat((Double) msgInfoBuckets.query("/1/message.dayOfWeek@NESTED/sumDay/value"), + closeTo(10.0, 0.01)); + } + + @Test + public void nestedFiledIsNotNull() throws IOException { + String sql = "SELECT e.name " + + "FROM elasticsearch-sql_test_index_employee_nested as e, e.projects as p " + + "WHERE p IS NOT NULL"; + + assertThat( + executeQuery(sql), + hitAll( + kvString("/_source/name", Is.is("Bob Smith")), + kvString("/_source/name", Is.is("Jane Smith")) + ) + ); + } + + // Doesn't support: aggregate function other than COUNT() + @SuppressWarnings("unused") + public void groupByNestedFieldAndAvg() throws IOException { + query( + "SELECT m.info, AVG(m.dayOfWeek)", + "GROUP BY m.info" + ); + query( + "SELECT m.info, AVG(myNum)", + "GROUP BY m.info" + ); + } + + @Test + public void groupByNestedAndRegularField() throws IOException { + final String sql = "SELECT someField, m.info, COUNT(*) " + FROM + " GROUP BY someField, m.info"; + + JSONObject result = executeQuery(sql); + JSONObject aggregation = getAggregation(result, "someField"); + JSONArray msgInfoBuckets = (JSONArray) aggregation.optQuery("/buckets"); + + Assert.assertNotNull(msgInfoBuckets); + Assert.assertThat(msgInfoBuckets.length(), equalTo(2)); + Assert.assertThat(msgInfoBuckets.query("/0/key"), equalTo("a")); + Assert.assertThat(msgInfoBuckets.query("/1/key"), equalTo("b")); + + JSONArray innerBuckets = + (JSONArray) msgInfoBuckets.optQuery("/0/message.info@NESTED/message.info/buckets"); + Assert.assertThat(innerBuckets.query("/0/key"), equalTo("b")); + Assert.assertThat(innerBuckets.query("/0/COUNT(*)/value"), equalTo(1)); + Assert.assertThat(innerBuckets.query("/1/key"), equalTo("c")); + Assert.assertThat(innerBuckets.query("/1/COUNT(*)/value"), equalTo(1)); + Assert.assertThat(innerBuckets.query("/2/key"), equalTo("zz")); + Assert.assertThat(innerBuckets.query("/2/COUNT(*)/value"), equalTo(1)); + + innerBuckets = + (JSONArray) msgInfoBuckets.optQuery("/1/message.info@NESTED/message.info/buckets"); + Assert.assertThat(innerBuckets.query("/0/key"), equalTo("a")); + Assert.assertThat(innerBuckets.query("/0/COUNT(*)/value"), equalTo(2)); + Assert.assertThat(innerBuckets.query("/1/key"), equalTo("c")); + Assert.assertThat(innerBuckets.query("/1/COUNT(*)/value"), equalTo(1)); + } + + @Test + public void countAggWithoutWhere() throws IOException { + String sql = "SELECT e.name, COUNT(p) as c " + + "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + + "GROUP BY e.name " + + "HAVING c > 1"; + + JSONObject result = executeQuery(sql); + JSONObject aggregation = getAggregation(result, "name.keyword"); + JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); + + Assert.assertNotNull(bucket); + Assert.assertThat(bucket.length(), equalTo(2)); + Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); + Assert.assertThat(bucket.query("/0/projects@NESTED/c/value"), equalTo(3)); + Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); + Assert.assertThat(bucket.query("/1/projects@NESTED/c/value"), equalTo(2)); + } + + @Test + public void countAggWithWhereOnParent() throws IOException { + String sql = "SELECT e.name, COUNT(p) as c " + + "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + + "WHERE e.name like '%smith%' " + + "GROUP BY e.name " + + "HAVING c > 1"; + + JSONObject result = executeQuery(sql); + JSONObject aggregation = getAggregation(result, "name.keyword"); + JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); + + Assert.assertNotNull(bucket); + Assert.assertThat(bucket.length(), equalTo(2)); + Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); + Assert.assertThat(bucket.query("/0/projects@NESTED/c/value"), equalTo(3)); + Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); + Assert.assertThat(bucket.query("/1/projects@NESTED/c/value"), equalTo(2)); + } + + @Test + public void countAggWithWhereOnNested() throws IOException { + String sql = "SELECT e.name, COUNT(p) as c " + + "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + + "WHERE p.name LIKE '%security%' " + + "GROUP BY e.name " + + "HAVING c > 1"; + + JSONObject result = executeQuery(sql); + JSONObject aggregation = getAggregation(result, "name.keyword"); + JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); + + Assert.assertNotNull(bucket); + Assert.assertThat(bucket.length(), equalTo(2)); + Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); + Assert.assertThat(bucket.query("/0/projects@NESTED/projects@FILTER/c/value"), equalTo(2)); + Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); + Assert.assertThat(bucket.query("/1/projects@NESTED/projects@FILTER/c/value"), equalTo(2)); + } + + @Test + public void countAggWithWhereOnParentOrNested() throws IOException { + String sql = "SELECT e.name, COUNT(p) as c " + + "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + + "WHERE e.name like '%smith%' or p.name LIKE '%security%' " + + "GROUP BY e.name " + + "HAVING c > 1"; + + JSONObject result = executeQuery(sql); + JSONObject aggregation = getAggregation(result, "name.keyword"); + JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); + + Assert.assertNotNull(bucket); + Assert.assertThat(bucket.length(), equalTo(2)); + Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); + Assert.assertThat(bucket.query("/0/projects@NESTED/c/value"), equalTo(3)); + Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); + Assert.assertThat(bucket.query("/1/projects@NESTED/c/value"), equalTo(2)); + } + + @Test + public void countAggWithWhereOnParentAndNested() throws IOException { + String sql = "SELECT e.name, COUNT(p) as c " + + "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + + "WHERE e.name like '%smith%' AND p.name LIKE '%security%' " + + "GROUP BY e.name " + + "HAVING c > 1"; + + JSONObject result = executeQuery(sql); + JSONObject aggregation = getAggregation(result, "name.keyword"); + JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); + + Assert.assertNotNull(bucket); + Assert.assertThat(bucket.length(), equalTo(2)); + Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); + Assert.assertThat(bucket.query("/0/projects@NESTED/projects@FILTER/c/value"), equalTo(2)); + Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); + Assert.assertThat(bucket.query("/1/projects@NESTED/projects@FILTER/c/value"), equalTo(2)); + } + + @Test + public void countAggWithWhereOnNestedAndNested() throws IOException { + String sql = "SELECT e.name, COUNT(p) as c " + + "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + + "WHERE p.started_year > 2000 AND p.name LIKE '%security%' " + + "GROUP BY e.name " + + "HAVING c > 0"; + + JSONObject result = executeQuery(sql); + JSONObject aggregation = getAggregation(result, "name.keyword"); + JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); + + Assert.assertNotNull(bucket); + Assert.assertThat(bucket.length(), equalTo(2)); + Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); + Assert.assertThat(bucket.query("/0/projects@NESTED/projects@FILTER/c/value"), equalTo(1)); + Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); + Assert.assertThat(bucket.query("/1/projects@NESTED/projects@FILTER/c/value"), equalTo(1)); + } + + @Test + public void countAggWithWhereOnNestedOrNested() throws IOException { + String sql = "SELECT e.name, COUNT(p) as c " + + "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + + "WHERE p.started_year > 2000 OR p.name LIKE '%security%' " + + "GROUP BY e.name " + + "HAVING c > 1"; + + JSONObject result = executeQuery(sql); + JSONObject aggregation = getAggregation(result, "name.keyword"); + JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); + + Assert.assertNotNull(bucket); + Assert.assertThat(bucket.length(), equalTo(2)); + Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); + Assert.assertThat(bucket.query("/0/projects@NESTED/projects@FILTER/c/value"), equalTo(2)); + Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); + Assert.assertThat(bucket.query("/1/projects@NESTED/projects@FILTER/c/value"), equalTo(2)); + } + + @Test + public void countAggOnNestedInnerFieldWithoutWhere() throws IOException { + String sql = "SELECT e.name, COUNT(p.started_year) as count " + + "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + + "WHERE p.name LIKE '%security%' " + + "GROUP BY e.name " + + "HAVING count > 0"; + + JSONObject result = executeQuery(sql); + JSONObject aggregation = getAggregation(result, "name.keyword"); + JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); + + Assert.assertNotNull(bucket); + Assert.assertThat(bucket.length(), equalTo(2)); + Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); + Assert.assertThat( + bucket.query("/0/projects.started_year@NESTED/projects.started_year@FILTER/count/value"), + equalTo(2)); + Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); + Assert.assertThat( + bucket.query("/1/projects.started_year@NESTED/projects.started_year@FILTER/count/value"), + equalTo(2)); + } + + @Test + public void maxAggOnNestedInnerFieldWithoutWhere() throws IOException { + String sql = "SELECT e.name, MAX(p.started_year) as max " + + "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + + "WHERE p.name LIKE '%security%' " + + "GROUP BY e.name"; + + JSONObject result = executeQuery(sql); + JSONObject aggregation = getAggregation(result, "name.keyword"); + JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); + + Assert.assertNotNull(bucket); + Assert.assertThat(bucket.length(), equalTo(2)); + Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); + Assert.assertThat( + bucket.query("/0/projects.started_year@NESTED/projects.started_year@FILTER/max/value"), + equalTo(2015.0)); + Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); + Assert.assertThat( + bucket.query("/1/projects.started_year@NESTED/projects.started_year@FILTER/max/value"), + equalTo(2015.0)); + } + + @Test + public void havingCountAggWithoutWhere() throws IOException { + String sql = "SELECT e.name " + + "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + + "GROUP BY e.name " + + "HAVING COUNT(p) > 1"; + + JSONObject result = executeQuery(sql); + JSONObject aggregation = getAggregation(result, "name.keyword"); + JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); + + Assert.assertNotNull(bucket); + Assert.assertThat(bucket.length(), equalTo(2)); + Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); + Assert.assertThat(bucket.query("/0/projects@NESTED/count_0/value"), equalTo(3)); + Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); + Assert.assertThat(bucket.query("/1/projects@NESTED/count_0/value"), equalTo(2)); + } + + @Test + public void havingCountAggWithWhereOnParent() throws IOException { + String sql = "SELECT e.name " + + "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + + "WHERE e.name like '%smith%' " + + "GROUP BY e.name " + + "HAVING COUNT(p) > 1"; + + JSONObject result = executeQuery(sql); + JSONObject aggregation = getAggregation(result, "name.keyword"); + JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); + + Assert.assertNotNull(bucket); + Assert.assertThat(bucket.length(), equalTo(2)); + Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); + Assert.assertThat(bucket.query("/0/projects@NESTED/count_0/value"), equalTo(3)); + Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); + Assert.assertThat(bucket.query("/1/projects@NESTED/count_0/value"), equalTo(2)); + } + + @Test + public void havingCountAggWithWhereOnNested() throws IOException { + String sql = "SELECT e.name " + + "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + + "WHERE p.name LIKE '%security%' " + + "GROUP BY e.name " + + "HAVING COUNT(p) > 1"; + + JSONObject result = executeQuery(sql); + JSONObject aggregation = getAggregation(result, "name.keyword"); + JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); + + Assert.assertNotNull(bucket); + Assert.assertThat(bucket.length(), equalTo(2)); + Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); + Assert.assertThat(bucket.query("/0/projects@NESTED/projects@FILTER/count_0/value"), equalTo(2)); + Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); + Assert.assertThat(bucket.query("/1/projects@NESTED/projects@FILTER/count_0/value"), equalTo(2)); + } + + @Test + public void havingCountAggWithWhereOnParentOrNested() throws IOException { + String sql = "SELECT e.name " + + "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + + "WHERE e.name like '%smith%' or p.name LIKE '%security%' " + + "GROUP BY e.name " + + "HAVING COUNT(p) > 1"; + + JSONObject result = executeQuery(sql); + JSONObject aggregation = getAggregation(result, "name.keyword"); + JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); + + Assert.assertNotNull(bucket); + Assert.assertThat(bucket.length(), equalTo(2)); + Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); + Assert.assertThat(bucket.query("/0/projects@NESTED/count_0/value"), equalTo(3)); + Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); + Assert.assertThat(bucket.query("/1/projects@NESTED/count_0/value"), equalTo(2)); + } + + @Test + public void havingCountAggWithWhereOnParentAndNested() throws IOException { + String sql = "SELECT e.name " + + "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + + "WHERE e.name like '%smith%' AND p.name LIKE '%security%' " + + "GROUP BY e.name " + + "HAVING COUNT(p) > 1"; + + JSONObject result = executeQuery(sql); + JSONObject aggregation = getAggregation(result, "name.keyword"); + JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); + + Assert.assertNotNull(bucket); + Assert.assertThat(bucket.length(), equalTo(2)); + Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); + Assert.assertThat(bucket.query("/0/projects@NESTED/projects@FILTER/count_0/value"), equalTo(2)); + Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); + Assert.assertThat(bucket.query("/1/projects@NESTED/projects@FILTER/count_0/value"), equalTo(2)); + } + + @Test + public void havingCountAggWithWhereOnNestedAndNested() throws IOException { + String sql = "SELECT e.name " + + "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + + "WHERE p.started_year > 2000 AND p.name LIKE '%security%' " + + "GROUP BY e.name " + + "HAVING COUNT(p) > 0"; + + JSONObject result = executeQuery(sql); + JSONObject aggregation = getAggregation(result, "name.keyword"); + JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); + + Assert.assertNotNull(bucket); + Assert.assertThat(bucket.length(), equalTo(2)); + Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); + Assert.assertThat(bucket.query("/0/projects@NESTED/projects@FILTER/count_0/value"), equalTo(1)); + Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); + Assert.assertThat(bucket.query("/1/projects@NESTED/projects@FILTER/count_0/value"), equalTo(1)); + } + + @Test + public void havingCountAggWithWhereOnNestedOrNested() throws IOException { + String sql = "SELECT e.name " + + "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + + "WHERE p.started_year > 2000 OR p.name LIKE '%security%' " + + "GROUP BY e.name " + + "HAVING COUNT(p) > 1"; + + JSONObject result = executeQuery(sql); + JSONObject aggregation = getAggregation(result, "name.keyword"); + JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); + + Assert.assertNotNull(bucket); + Assert.assertThat(bucket.length(), equalTo(2)); + Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); + Assert.assertThat(bucket.query("/0/projects@NESTED/projects@FILTER/count_0/value"), equalTo(2)); + Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); + Assert.assertThat(bucket.query("/1/projects@NESTED/projects@FILTER/count_0/value"), equalTo(2)); + } + + @Test + public void havingCountAggOnNestedInnerFieldWithoutWhere() throws IOException { + String sql = "SELECT e.name " + + "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + + "WHERE p.name LIKE '%security%' " + + "GROUP BY e.name " + + "HAVING COUNT(p.started_year) > 0"; + + JSONObject result = executeQuery(sql); + JSONObject aggregation = getAggregation(result, "name.keyword"); + JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); + + Assert.assertNotNull(bucket); + Assert.assertThat(bucket.length(), equalTo(2)); + Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); + Assert.assertThat( + bucket.query("/0/projects.started_year@NESTED/projects.started_year@FILTER/count_0/value"), + equalTo(2)); + Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); + Assert.assertThat( + bucket.query("/1/projects.started_year@NESTED/projects.started_year@FILTER/count_0/value"), + equalTo(2)); + } + + @Test + public void havingMaxAggOnNestedInnerFieldWithoutWhere() throws IOException { + String sql = "SELECT e.name " + + "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + + "WHERE p.name LIKE '%security%' " + + "GROUP BY e.name " + + "HAVING MAX(p.started_year) > 1990"; + + JSONObject result = executeQuery(sql); + JSONObject aggregation = getAggregation(result, "name.keyword"); + JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); + + Assert.assertNotNull(bucket); + Assert.assertThat(bucket.length(), equalTo(2)); + Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); + Assert.assertThat( + bucket.query("/0/projects.started_year@NESTED/projects.started_year@FILTER/max_0/value"), + equalTo(2015.0)); + Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); + Assert.assertThat( + bucket.query("/1/projects.started_year@NESTED/projects.started_year@FILTER/max_0/value"), + equalTo(2015.0)); + } + + /*********************************************************** + Matchers for Non-Aggregation Testing + ***********************************************************/ + + @SafeVarargs + private final Matcher hits(Matcher... subMatchers) { + return featureValueOf("hits", arrayContainingInAnyOrder(subMatchers), + resp -> resp.getHits().getHits()); + } + + @SafeVarargs + private final Matcher hit(Matcher... subMatchers) { + return allOf(subMatchers); + } + + private Matcher myNum(int value) { + return kv("myNum", is(value)); + } + + private Matcher myNum(int[] values) { + + return new BaseMatcher() { + + @Override + public boolean matches(Object item) { + + if (item instanceof SearchHit) { + final SearchHit hit = (SearchHit) item; + ArrayList actualValues = (ArrayList) hit.getSourceAsMap().get("myNum"); + + if (actualValues.size() != values.length) { + return false; + } + for (int i = 0; i < values.length; ++i) { + if (values[i] != actualValues.get(i)) { + return false; + } + } + return true; + } + + return false; + } + + @Override + public void describeTo(Description description) { + } + }; + } + + private Matcher someField(String value) { + return kv("someField", is(value)); + } + + private Matcher author(String value) { + return kv("author", is(value)); + } + + private Matcher info(String value) { + return kv("info", is(value)); + } + + private Matcher kv(String key, Matcher valMatcher) { + return featureValueOf(key, valMatcher, hit -> hit.getSourceAsMap().get(key)); + } + + @SafeVarargs + private final Matcher innerHits(String path, Matcher... innerHitMatchers) { + return featureValueOf( + "innerHits", + arrayContainingInAnyOrder(innerHitMatchers), + hit -> hit.getInnerHits().get(path).getHits() + ); + } + + /*********************************************************** + Matchers for Aggregation Testing + ***********************************************************/ + + private FeatureMatcher featureValueOf(String name, Matcher subMatcher, + Function getter) { + return new FeatureMatcher(subMatcher, name, name) { + @Override + protected U featureValueOf(T actual) { + return getter.apply(actual); + } + }; + } + + /*********************************************************** + Query Utility to Fetch Response for SQL + ***********************************************************/ + + private SearchResponse query(String select, String... statements) throws IOException { + return execute(select + " " + FROM + " " + String.join(" ", statements)); + } + + private SearchResponse execute(String sql) throws IOException { + final JSONObject jsonObject = executeQuery(sql); + + final XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser( + NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, + jsonObject.toString()); + return SearchResponse.fromXContent(parser); + } + + private JSONObject getAggregation(final JSONObject queryResult, final String aggregationName) { + final String aggregationsObjName = "aggregations"; + Assert.assertTrue(queryResult.has(aggregationsObjName)); + + final JSONObject aggregations = queryResult.getJSONObject(aggregationsObjName); + Assert.assertTrue(aggregations.has(aggregationName)); + return aggregations.getJSONObject(aggregationName); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/ODFERestTestCase.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/ODFERestTestCase.java new file mode 100644 index 0000000000..19063821c2 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/ODFERestTestCase.java @@ -0,0 +1,130 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import java.io.IOException; +import java.util.Map; +import java.util.Optional; +import org.apache.http.Header; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.message.BasicHeader; +import org.apache.http.ssl.SSLContextBuilder; +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.json.JSONArray; +import org.json.JSONObject; + +/** + * ODFE integration test base class to support both security disabled and enabled ODFE cluster. + */ +public abstract class ODFERestTestCase extends ESRestTestCase { + + protected boolean isHttps() { + boolean isHttps = Optional.ofNullable(System.getProperty("https")) + .map("true"::equalsIgnoreCase).orElse(false); + if (isHttps) { + //currently only external cluster is supported for security enabled testing + if (!Optional.ofNullable(System.getProperty("tests.rest.cluster")).isPresent()) { + throw new RuntimeException( + "external cluster url should be provided for security enabled testing"); + } + } + + return isHttps; + } + + protected String getProtocol() { + return isHttps() ? "https" : "http"; + } + + protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException { + RestClientBuilder builder = RestClient.builder(hosts); + if (isHttps()) { + configureHttpsClient(builder, settings); + } else { + configureClient(builder, settings); + } + + builder.setStrictDeprecationMode(true); + return builder.build(); + } + + protected static void wipeAllODFEIndices() throws IOException { + Response response = client().performRequest(new Request("GET", "/_cat/indices?format=json")); + JSONArray jsonArray = new JSONArray(EntityUtils.toString(response.getEntity(), "UTF-8")); + for (Object object : jsonArray) { + JSONObject jsonObject = (JSONObject) object; + String indexName = jsonObject.getString("index"); + //.opendistro_security isn't allowed to delete from cluster + if (!".opendistro_security".equals(indexName)) { + client().performRequest(new Request("DELETE", "/" + indexName)); + } + } + } + + protected static void configureHttpsClient(RestClientBuilder builder, Settings settings) + throws IOException { + Map headers = ThreadContext.buildDefaultHeaders(settings); + Header[] defaultHeaders = new Header[headers.size()]; + int i = 0; + for (Map.Entry entry : headers.entrySet()) { + defaultHeaders[i++] = new BasicHeader(entry.getKey(), entry.getValue()); + } + builder.setDefaultHeaders(defaultHeaders); + builder.setHttpClientConfigCallback(httpClientBuilder -> { + String userName = Optional.ofNullable(System.getProperty("user")) + .orElseThrow(() -> new RuntimeException("user name is missing")); + String password = Optional.ofNullable(System.getProperty("password")) + .orElseThrow(() -> new RuntimeException("password is missing")); + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider + .setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, password)); + try { + return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider) + //disable the certificate since our testing cluster just uses the default security configuration + .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .setSSLContext(SSLContextBuilder.create() + .loadTrustMaterial(null, (chains, authType) -> true) + .build()); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + final String socketTimeoutString = settings.get(CLIENT_SOCKET_TIMEOUT); + final TimeValue socketTimeout = + TimeValue.parseTimeValue(socketTimeoutString == null ? "60s" : socketTimeoutString, + CLIENT_SOCKET_TIMEOUT); + builder.setRequestConfigCallback( + conf -> conf.setSocketTimeout(Math.toIntExact(socketTimeout.getMillis()))); + if (settings.hasValue(CLIENT_PATH_PREFIX)) { + builder.setPathPrefix(settings.get(CLIENT_PATH_PREFIX)); + } + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/OrderIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/OrderIT.java new file mode 100644 index 0000000000..5d73983d75 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/OrderIT.java @@ -0,0 +1,161 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static org.hamcrest.Matchers.equalTo; + +import java.io.IOException; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Test; + +public class OrderIT extends SQLIntegTestCase { + + @Override + protected void init() throws Exception { + loadIndex(Index.ORDER); + } + + @Test + public void simpleOrder() throws IOException { + String query = "SELECT id, name FROM elasticsearch-sql_test_index_order ORDER BY id"; + JSONArray result = getSortExplain(query); + assertThat(result.length(), equalTo(1)); + JSONObject jsonObject = getSortByField(result, "id"); + assertThat(query(result, "/0/id/order"), equalTo("asc")); + assertFalse(jsonObject.getJSONObject("id").has("missing")); + } + + @Test + public void orderByScore() throws IOException { + String query = "SELECT * FROM elasticsearch-sql_test_index_order ORDER BY _score"; + JSONArray result = getSortExplain(query); + assertThat(result.length(), equalTo(1)); + JSONObject jsonObject = getSortByField(result, "_score"); + assertThat(query(result, "/0/_score/order"), equalTo("asc")); + assertFalse(jsonObject.getJSONObject("_score").has("missing")); + + JSONObject response = executeQuery(query); + JSONArray hits = getHits(response); + assertThat(hits.length(), equalTo(7)); + } + + @Test + public void simpleOrderMultipleFields() throws IOException { + String query = "SELECT id, name FROM elasticsearch-sql_test_index_order ORDER BY id, name"; + JSONArray result = getSortExplain(query); + assertThat(result.length(), equalTo(2)); + assertTrue(result.getJSONObject(0).has("id")); + assertTrue(result.getJSONObject(1).has("name.keyword")); + } + + @Test + public void explicitOrderType() throws IOException { + String query = + "SELECT id, name FROM elasticsearch-sql_test_index_order ORDER BY id ASC, name DESC"; + JSONArray result = getSortExplain(query); + assertThat(result.length(), equalTo(2)); + assertThat(query(result, "/0/id/order"), equalTo("asc")); + assertThat(query(result, "/1/name.keyword/order"), equalTo("desc")); + } + + @Test + public void orderByIsNull() throws IOException { + String query = "SELECT * FROM elasticsearch-sql_test_index_order ORDER BY id IS NULL, id DESC"; + JSONArray result = getSortExplain(query); + assertThat(result.length(), equalTo(1)); + assertThat(query(result, "/0/id/order"), equalTo("desc")); + assertThat(query(result, "/0/id/missing"), equalTo("_last")); + + JSONObject response = executeQuery(query); + JSONArray hits = getHits(response); + assertThat(query(hits, "/0/_source/id"), equalTo("5")); + + // Another equivalent syntax + assertThat(explainQuery("SELECT * FROM elasticsearch-sql_test_index_order " + + "ORDER BY id IS NULL, id DESC"), + equalTo(explainQuery("SELECT * FROM elasticsearch-sql_test_index_order " + + "ORDER BY id IS NULL DESC")) + ); + } + + @Test + public void orderByIsNotNull() throws IOException { + String query = + "SELECT id, name FROM elasticsearch-sql_test_index_order ORDER BY name IS NOT NULL"; + JSONArray result = getSortExplain(query); + assertThat(1, equalTo(result.length())); + assertThat(query(result, "/0/name.keyword/order"), equalTo("asc")); + assertThat(query(result, "/0/name.keyword/missing"), equalTo("_first")); + + JSONObject response = executeQuery(query); + JSONArray hits = getHits(response); + assertFalse(hits.getJSONObject(0).getJSONObject("_source").has("name")); + assertThat(hits.getJSONObject(hits.length() - 1).query("/_source/name").toString(), + equalTo("f")); + + // Another equivalent syntax + assertThat(explainQuery("SELECT id, name FROM elasticsearch-sql_test_index_order " + + "ORDER BY name IS NOT NULL"), + equalTo(explainQuery("SELECT id, name FROM elasticsearch-sql_test_index_order " + + "ORDER BY name IS NOT NULL ASC")) + ); + } + + @Test + public void multipleOrderByWithNulls() throws IOException { + String query = + "SELECT id, name FROM elasticsearch-sql_test_index_order ORDER BY id IS NULL, name IS NOT NULL"; + JSONArray result = getSortExplain(query); + assertThat(result.length(), equalTo(2)); + assertThat(query(result, "/0/id/missing"), equalTo("_last")); + assertThat(query(result, "/1/name.keyword/missing"), equalTo("_first")); + } + + @Test + public void testOrderByMergeForSameField() throws IOException { + String query = "SELECT * FROM elasticsearch-sql_test_index_order " + + "ORDER BY id IS NULL, name DESC, id DESC, id IS NOT NULL, name IS NULL"; + JSONArray result = getSortExplain(query); + assertThat(2, equalTo(result.length())); + assertThat(query(result, "/0/id/order"), equalTo("asc")); + assertThat(query(result, "/0/id/missing"), equalTo("_first")); + assertThat(query(result, "/1/name.keyword/order"), equalTo("asc")); + assertThat(query(result, "/1/name.keyword/missing"), equalTo("_last")); + } + + private JSONArray getSortExplain(String sqlQuery) throws IOException { + String result = explainQuery(sqlQuery); + JSONObject jsonObject = new JSONObject(result); + return jsonObject.getJSONArray("sort"); + } + + private JSONObject getSortByField(JSONArray sortArray, String orderByName) { + JSONObject jsonObject; + for (int i = 0; i < sortArray.length(); i++) { + jsonObject = (JSONObject) sortArray.get(i); + if (jsonObject.has(orderByName)) { + return jsonObject; + } + } + return null; + } + + private String query(JSONArray jsonArray, String jsonPath) { + return jsonArray.query(jsonPath).toString(); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/OrdinalAliasRewriterIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/OrdinalAliasRewriterIT.java new file mode 100644 index 0000000000..d37cf891f6 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/OrdinalAliasRewriterIT.java @@ -0,0 +1,169 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static org.hamcrest.Matchers.equalTo; + +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; +import java.io.IOException; +import org.junit.Test; + +public class OrdinalAliasRewriterIT extends SQLIntegTestCase { + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + } + + // tests query results with jdbc output + @Test + public void simpleGroupByOrdinal() { + String expected = executeQuery(StringUtils.format( + "SELECT lastname FROM %s AS b GROUP BY lastname LIMIT 3", + TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); + String actual = executeQuery(StringUtils.format( + "SELECT lastname FROM %s AS b GROUP BY 1 LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), + "jdbc"); + assertThat(actual, equalTo(expected)); + } + + @Test + public void multipleGroupByOrdinal() { + String expected = executeQuery(StringUtils.format( + "SELECT lastname, firstname, age FROM %s AS b GROUP BY firstname, age, lastname LIMIT 3", + TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); + String actual = executeQuery(StringUtils.format( + "SELECT lastname, firstname, age FROM %s AS b GROUP BY 2, 3, 1 LIMIT 3", + TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); + assertThat(actual, equalTo(expected)); + } + + @Test + public void selectFieldiWithBacticksGroupByOrdinal() { + String expected = executeQuery(StringUtils.format( + "SELECT `lastname` FROM %s AS b GROUP BY `lastname` LIMIT 3", + TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); + String actual = executeQuery(StringUtils.format( + "SELECT `lastname` FROM %s AS b GROUP BY 1 LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), + "jdbc"); + assertThat(actual, equalTo(expected)); + } + + @Test + public void selectFieldiWithBacticksAndTableAliasGroupByOrdinal() { + String expected = executeQuery(StringUtils.format( + "SELECT `b`.`lastname`, `age`, firstname FROM %s AS b GROUP BY `age`, `b`.`lastname` , firstname LIMIT 10", + TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); + String actual = executeQuery(StringUtils.format( + "SELECT `b`.`lastname`, `age`, firstname FROM %s AS b GROUP BY 2, 1, 3 LIMIT 10", + TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); + assertThat(actual, equalTo(expected)); + } + + @Test + public void simpleOrderByOrdinal() { + String expected = executeQuery(StringUtils.format( + "SELECT lastname FROM %s AS b ORDER BY lastname LIMIT 3", + TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); + String actual = executeQuery(StringUtils.format( + "SELECT lastname FROM %s AS b ORDER BY 1 LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), + "jdbc"); + assertThat(actual, equalTo(expected)); + } + + @Test + public void multipleOrderByOrdinal() { + String expected = executeQuery(StringUtils.format( + "SELECT lastname, firstname, age FROM %s AS b ORDER BY firstname, age, lastname LIMIT 3", + TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); + String actual = executeQuery(StringUtils.format( + "SELECT lastname, firstname, age FROM %s AS b ORDER BY 2, 3, 1 LIMIT 3", + TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); + assertThat(actual, equalTo(expected)); + } + + @Test + public void selectFieldiWithBacticksOrderByOrdinal() { + String expected = executeQuery(StringUtils.format( + "SELECT `lastname` FROM %s AS b ORDER BY `lastname` LIMIT 3", + TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); + String actual = executeQuery(StringUtils.format( + "SELECT `lastname` FROM %s AS b ORDER BY 1 LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), + "jdbc"); + assertThat(actual, equalTo(expected)); + } + + @Test + public void selectFieldiWithBacticksAndTableAliasOrderByOrdinal() { + String expected = executeQuery(StringUtils.format( + "SELECT `b`.`lastname` FROM %s AS b ORDER BY `b`.`lastname` LIMIT 3", + TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); + String actual = executeQuery(StringUtils.format( + "SELECT `b`.`lastname` FROM %s AS b ORDER BY 1 LIMIT 3", + TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); + assertThat(actual, equalTo(expected)); + } + + // ORDER BY IS NULL/NOT NULL + @Test + public void selectFieldiWithBacticksAndTableAliasOrderByOrdinalAndNull() { + String expected = executeQuery(StringUtils.format( + "SELECT `b`.`lastname`, age FROM %s AS b ORDER BY `b`.`lastname` IS NOT NULL DESC, age is NULL LIMIT 3", + TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); + String actual = executeQuery(StringUtils.format( + "SELECT `b`.`lastname`, age FROM %s AS b ORDER BY 1 IS NOT NULL DESC, 2 IS NULL LIMIT 3", + TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); + assertThat(actual, equalTo(expected)); + } + + + // explain + @Test + public void explainSelectFieldiWithBacticksAndTableAliasGroupByOrdinal() throws IOException { + String expected = explainQuery(StringUtils.format( + "SELECT `b`.`lastname` FROM %s AS b GROUP BY `b`.`lastname` LIMIT 3", + TestsConstants.TEST_INDEX_ACCOUNT)); + String actual = explainQuery(StringUtils.format( + "SELECT `b`.`lastname` FROM %s AS b GROUP BY 1 LIMIT 3", + TestsConstants.TEST_INDEX_ACCOUNT)); + assertThat(actual, equalTo(expected)); + } + + @Test + public void explainSelectFieldiWithBacticksAndTableAliasOrderByOrdinal() throws IOException { + String expected = explainQuery(StringUtils.format( + "SELECT `b`.`lastname` FROM %s AS b ORDER BY `b`.`lastname` LIMIT 3", + TestsConstants.TEST_INDEX_ACCOUNT)); + String actual = explainQuery(StringUtils.format( + "SELECT `b`.`lastname` FROM %s AS b ORDER BY 1 LIMIT 3", + TestsConstants.TEST_INDEX_ACCOUNT)); + assertThat(actual, equalTo(expected)); + } + + // explain ORDER BY IS NULL/NOT NULL + @Test + public void explainSelectFieldiWithBacticksAndTableAliasOrderByOrdinalAndNull() + throws IOException { + String expected = explainQuery(StringUtils.format( + "SELECT `b`.`lastname`, age FROM %s AS b ORDER BY `b`.`lastname` IS NOT NULL DESC, age is NULL LIMIT 3", + TestsConstants.TEST_INDEX_ACCOUNT)); + String actual = explainQuery(StringUtils.format( + "SELECT `b`.`lastname`, age FROM %s AS b ORDER BY 1 IS NOT NULL DESC, 2 IS NULL LIMIT 3", + TestsConstants.TEST_INDEX_ACCOUNT)); + assertThat(actual, equalTo(expected)); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PluginIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PluginIT.java new file mode 100644 index 0000000000..82d24ace41 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PluginIT.java @@ -0,0 +1,453 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getResponseBody; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static org.hamcrest.Matchers.equalTo; + +import java.io.IOException; +import java.util.Locale; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.json.JSONObject; +import org.junit.Test; + +public class PluginIT extends SQLIntegTestCase { + + @Override + protected void init() throws Exception { + wipeAllClusterSettings(); + } + + @Test + public void sqlEnableSettingsTest() throws IOException { + loadIndex(Index.ACCOUNT); + updateClusterSettings(new ClusterSetting(PERSISTENT, "opendistro.sql.enabled", "true")); + String query = String + .format(Locale.ROOT, "SELECT firstname FROM %s WHERE account_number=1", TEST_INDEX_ACCOUNT); + JSONObject queryResult = executeQuery(query); + assertThat(getHits(queryResult).length(), equalTo(1)); + + updateClusterSettings(new ClusterSetting(PERSISTENT, "opendistro.sql.enabled", "false")); + Response response = null; + try { + queryResult = executeQuery(query); + } catch (ResponseException ex) { + response = ex.getResponse(); + } + + queryResult = new JSONObject(TestUtils.getResponseBody(response)); + assertThat(queryResult.getInt("status"), equalTo(400)); + JSONObject error = queryResult.getJSONObject("error"); + assertThat(error.getString("reason"), equalTo("Invalid SQL query")); + assertThat(error.getString("details"), equalTo( + "Either opendistro.sql.enabled or rest.action.multi.allow_explicit_index setting is false")); + assertThat(error.getString("type"), equalTo("SQLFeatureDisabledException")); + wipeAllClusterSettings(); + } + + @Test + public void sqlTransientOnlySettingTest() throws IOException { + // (1) compact form + String settings = "{" + + " \"transient\": {" + + " \"opendistro.sql.metrics.rollinginterval\": \"80\"" + + " }" + + "}"; + JSONObject actual = updateViaSQLSettingsAPI(settings); + JSONObject expected = new JSONObject("{" + + " \"acknowledged\" : true," + + " \"persistent\" : { }," + + " \"transient\" : {" + + " \"opendistro\" : {" + + " \"sql\" : {" + + " \"metrics\" : {" + + " \"rollinginterval\" : \"80\"" + + " }" + + " }" + + " }" + + " }" + + "}"); + assertTrue(actual.similar(expected)); + + // (2) partial expanded form + settings = "{" + + " \"transient\": {" + + " \"opendistro\" : {" + + " \"sql\" : {" + + " \"metrics.rollinginterval\": \"75\"" + + " }" + + " }" + + " }" + + "}"; + actual = updateViaSQLSettingsAPI(settings); + expected = new JSONObject("{" + + " \"acknowledged\" : true," + + " \"persistent\" : { }," + + " \"transient\" : {" + + " \"opendistro\" : {" + + " \"sql\" : {" + + " \"metrics\" : {" + + " \"rollinginterval\" : \"75\"" + + " }" + + " }" + + " }" + + " }" + + "}"); + assertTrue(actual.similar(expected)); + + + // (3) full expanded form + settings = "{" + + " \"transient\": {" + + " \"opendistro\" : {" + + " \"sql\" : {" + + " \"metrics\": {" + + " \"rollinginterval\": \"65\"" + + " }" + + " }" + + " }" + + " }" + + "}"; + actual = updateViaSQLSettingsAPI(settings); + expected = new JSONObject("{" + + " \"acknowledged\" : true," + + " \"persistent\" : { }," + + " \"transient\" : {" + + " \"opendistro\" : {" + + " \"sql\" : {" + + " \"metrics\" : {" + + " \"rollinginterval\" : \"65\"" + + " }" + + " }" + + " }" + + " }" + + "}"); + assertTrue(actual.similar(expected)); + } + + @Test + public void sqlPersistentOnlySettingTest() throws IOException { + // (1) compact form + String settings = "{" + + " \"persistent\": {" + + " \"opendistro.sql.metrics.rollinginterval\": \"80\"" + + " }" + + "}"; + JSONObject actual = updateViaSQLSettingsAPI(settings); + JSONObject expected = new JSONObject("{" + + " \"acknowledged\" : true," + + " \"transient\" : { }," + + " \"persistent\" : {" + + " \"opendistro\" : {" + + " \"sql\" : {" + + " \"metrics\" : {" + + " \"rollinginterval\" : \"80\"" + + " }" + + " }" + + " }" + + " }" + + "}"); + assertTrue(actual.similar(expected)); + + // (2) partial expanded form + settings = "{" + + " \"persistent\": {" + + " \"opendistro\" : {" + + " \"sql\" : {" + + " \"metrics.rollinginterval\": \"75\"" + + " }" + + " }" + + " }" + + "}"; + actual = updateViaSQLSettingsAPI(settings); + expected = new JSONObject("{" + + " \"acknowledged\" : true," + + " \"transient\" : { }," + + " \"persistent\" : {" + + " \"opendistro\" : {" + + " \"sql\" : {" + + " \"metrics\" : {" + + " \"rollinginterval\" : \"75\"" + + " }" + + " }" + + " }" + + " }" + + "}"); + assertTrue(actual.similar(expected)); + + + // (3) full expanded form + settings = "{" + + " \"persistent\": {" + + " \"opendistro\" : {" + + " \"sql\" : {" + + " \"metrics\": {" + + " \"rollinginterval\": \"65\"" + + " }" + + " }" + + " }" + + " }" + + "}"; + actual = updateViaSQLSettingsAPI(settings); + expected = new JSONObject("{" + + " \"acknowledged\" : true," + + " \"transient\" : { }," + + " \"persistent\" : {" + + " \"opendistro\" : {" + + " \"sql\" : {" + + " \"metrics\" : {" + + " \"rollinginterval\" : \"65\"" + + " }" + + " }" + + " }" + + " }" + + "}"); + assertTrue(actual.similar(expected)); + } + + /** + * Both transient and persistent settings are applied for same settings. + * This is similiar to _cluster/settings behavior + */ + @Test + public void sqlCombinedSettingTest() throws IOException { + String settings = "{" + + " \"transient\": {" + + " \"opendistro.sql.metrics.rollingwindow\": \"3700\"," + + " \"opendistro.sql.query.analysis.semantic.suggestion\" : \"false\"" + + " }," + + " \"persistent\": {" + + " \"opendistro.sql.query.analysis.semantic.suggestion\" : \"true\"" + + " }" + + "}"; + JSONObject actual = updateViaSQLSettingsAPI(settings); + JSONObject expected = new JSONObject("{" + + " \"acknowledged\" : true," + + " \"persistent\" : {" + + " \"opendistro\" : {" + + " \"sql\" : {" + + " \"query\" : {" + + " \"analysis\" : {" + + " \"semantic\" : {" + + " \"suggestion\" : \"true\"" + + " }" + + " }" + + " }" + + " }" + + " }" + + " }," + + " \"transient\" : {" + + " \"opendistro\" : {" + + " \"sql\" : {" + + " \"metrics\" : {" + + " \"rollingwindow\" : \"3700\"" + + " }," + + " \"query\" : {" + + " \"analysis\" : {" + + " \"semantic\" : {" + + " \"suggestion\" : \"false\"" + + " }" + + " }" + + " }" + + " }" + + " }" + + " }" + + "}"); + assertTrue(actual.similar(expected)); + } + + /** + * Ignore all non opendistro.sql settings. + * Only settings starting with opendistro.sql. are affected + */ + @Test + public void ignoreNonSQLSettingsTest() throws IOException { + String settings = "{" + + " \"transient\": {" + + " \"opendistro.sql.metrics.rollingwindow\": \"3700\"," + + " \"opendistro.alerting.metrics.rollingwindow\": \"3700\"," + + " \"search.max_buckets\": \"10000\"," + + " \"search.max_keep_alive\": \"24h\"" + + " }," + + " \"persistent\": {" + + " \"opendistro.sql.query.analysis.semantic.suggestion\": \"true\"," + + " \"opendistro.alerting.metrics.rollingwindow\": \"3700\"," + + " \"thread_pool.analyze.queue_size\": \"16\"" + + " }" + + "}"; + JSONObject actual = updateViaSQLSettingsAPI(settings); + JSONObject expected = new JSONObject("{" + + " \"acknowledged\" : true," + + " \"persistent\" : {" + + " \"opendistro\" : {" + + " \"sql\" : {" + + " \"query\" : {" + + " \"analysis\" : {" + + " \"semantic\" : {" + + " \"suggestion\" : \"true\"" + + " }" + + " }" + + " }" + + " }" + + " }" + + " }," + + " \"transient\" : {" + + " \"opendistro\" : {" + + " \"sql\" : {" + + " \"metrics\" : {" + + " \"rollingwindow\" : \"3700\"" + + " }" + + " }" + + " }" + + " }" + + "}"); + assertTrue(actual.similar(expected)); + } + + @Test + public void ignoreNonTransientNonPersistentSettingsTest() throws IOException { + String settings = "{" + + " \"transient\": {" + + " \"opendistro.sql.query.response.format\": \"jdbc\"" + + " }," + + " \"persistent\": {" + + " \"opendistro.sql.query.slowlog\": \"2\"" + + " }," + + " \"hello\": {" + + " \"world\" : {" + + " \"name\" : \"John Doe\"" + + " }" + + " }" + + "}"; + JSONObject actual = updateViaSQLSettingsAPI(settings); + JSONObject expected = new JSONObject("{" + + " \"acknowledged\" : true," + + " \"persistent\" : {" + + " \"opendistro\" : {" + + " \"sql\" : {" + + " \"query\" : {" + + " \"slowlog\" : \"2\"" + + " }" + + " }" + + " }" + + " }," + + " \"transient\" : {" + + " \"opendistro\" : {" + + " \"sql\" : {" + + " \"query\" : {" + + " \"response\" : {" + + " \"format\" : \"jdbc\"" + + " }" + + " }" + + " }" + + " }" + + " }" + + "}"); + assertTrue(actual.similar(expected)); + } + + @Test + public void sqlCombinedMixedSettingTest() throws IOException { + String settings = "{" + + " \"transient\": {" + + " \"opendistro.sql.query.response.format\": \"json\"" + + " }," + + " \"persistent\": {" + + " \"opendistro\": {" + + " \"sql\": {" + + " \"query\": {" + + " \"slowlog\": \"1\"," + + " \"response.format\": \"jdbc\"" + + " }" + + " }" + + " }" + + " }," + + " \"hello\": {" + + " \"world\": {" + + " \"city\": \"Seattle\"" + + " }" + + " }" + + "}"; + JSONObject actual = updateViaSQLSettingsAPI(settings); + JSONObject expected = new JSONObject("{" + + " \"acknowledged\" : true," + + " \"persistent\" : {" + + " \"opendistro\" : {" + + " \"sql\" : {" + + " \"query\" : {" + + " \"slowlog\" : \"1\"," + + " \"response\" : {" + + " \"format\" : \"jdbc\"" + + " }" + + " }" + + " }" + + " }" + + " }," + + " \"transient\" : {" + + " \"opendistro\" : {" + + " \"sql\" : {" + + " \"query\" : {" + + " \"response\" : {" + + " \"format\" : \"json\"" + + " }" + + " }" + + " }" + + " }" + + " }" + + "}"); + assertTrue(actual.similar(expected)); + } + + @Test + public void nonRegisteredSQLSettingsThrowException() throws IOException { + String settings = "{" + + " \"transient\": {" + + " \"opendistro.sql.query.state.city\": \"Seattle\"" + + " }" + + "}"; + + JSONObject actual; + Response response = null; + try { + actual = updateViaSQLSettingsAPI(settings); + } catch (ResponseException ex) { + response = ex.getResponse(); + } + + actual = new JSONObject(TestUtils.getResponseBody(response)); + assertThat(actual.getInt("status"), equalTo(400)); + assertThat(actual.query("/error/type"), equalTo("illegal_argument_exception")); + assertThat( + actual.query("/error/reason"), + equalTo("transient setting [opendistro.sql.query.state.city], not recognized") + ); + } + + protected static JSONObject updateViaSQLSettingsAPI(String body) throws IOException { + Request request = new Request("PUT", "/_opendistro/_sql/settings"); + request.setJsonEntity(body); + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + request.setOptions(restOptionsBuilder); + Response response = client().performRequest(request); + return new JSONObject(getResponseBody(response)); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PreparedStatementIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PreparedStatementIT.java new file mode 100644 index 0000000000..ec4501eecf --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PreparedStatementIT.java @@ -0,0 +1,90 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import java.io.IOException; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Assert; +import org.junit.Test; + +// Refer to https://www.elastic.co/guide/en/elasticsearch/reference/6.5/integration-tests.html +// for detailed ESIntegTestCase usages doc. +public class PreparedStatementIT extends SQLIntegTestCase { + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + } + + @Test + public void testPreparedStatement() throws IOException { + int ageToCompare = 35; + + JSONObject response = executeRequest(String.format("{\n" + + " \"query\": \"SELECT * FROM %s WHERE age > ? AND state in (?, ?) LIMIT ?\",\n" + + " \"parameters\": [\n" + + " {\n" + + " \"type\": \"integer\",\n" + + " \"value\": \"" + ageToCompare + "\"\n" + + " },\n" + + " {\n" + + " \"type\": \"string\",\n" + + " \"value\": \"TN\"\n" + + " },\n" + + " {\n" + + " \"type\": \"string\",\n" + + " \"value\": \"UT\"\n" + + " },\n" + + " {\n" + + " \"type\": \"integer\",\n" + + " \"value\": \"20\"\n" + + " }\n" + + " ]\n" + + "}", TestsConstants.TEST_INDEX_ACCOUNT)); + + Assert.assertTrue(response.has("hits")); + Assert.assertTrue(response.getJSONObject("hits").has("hits")); + + JSONArray hits = response.getJSONObject("hits").getJSONArray("hits"); + Assert.assertTrue(hits.length() > 0); + for (int i = 0; i < hits.length(); i++) { + JSONObject accountJson = hits.getJSONObject(i); + Assert.assertTrue(accountJson.getJSONObject("_source").getInt("age") > ageToCompare); + } + } + + /* currently the integ test case will fail if run using Intellj, have to run using gradle command + * because the integ test cluster created by IntellJ has http diabled, need to spend some time later to + * figure out how to configure the integ test cluster properly. Related online resources: + * https://discuss.elastic.co/t/http-enabled-with-esintegtestcase/102032 + * https://discuss.elastic.co/t/help-with-esintegtestcase/105245 + @Override + protected Collection> nodePlugins() { + return Arrays.asList(MockTcpTransportPlugin.class); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder().put(super.nodeSettings(nodeOrdinal)) + // .put("node.mode", "network") + .put("http.enabled", true) + //.put("http.type", "netty4") + .build(); + } + */ +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java new file mode 100644 index 0000000000..cb5be363dc --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java @@ -0,0 +1,612 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static java.util.stream.Collectors.toSet; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; + +import com.google.common.collect.Sets; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; +import org.elasticsearch.client.Request; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Ignore; +import org.junit.Test; + +/** + * PrettyFormatResponseIT will likely be excluding some of the tests written in PrettyFormatResponseTest since + * those tests were asserting on class objects directly. These updated tests will only be making assertions based + * on the REST response. + *

+ * Any integ tests from PrettyFormatResponseTest that were excluded can perhaps later be changed and moved over + * to be unit tests. + *

+ * Tests from original integ tests excluded: + * - noIndexType() + * - withIndexType() + */ +public class PrettyFormatResponseIT extends SQLIntegTestCase { + + private static final Set allAccountFields = Sets.newHashSet( + "account_number", "balance", "firstname", "lastname", "age", "gender", "address", "employer", + "email", "city", "state" + ); + + private static final Set regularFields = Sets.newHashSet("someField", "myNum"); + + private static final Set messageFields = Sets.newHashSet( + "message.dayOfWeek", "message.info", "message.author"); + + private static final Set commentFields = Sets.newHashSet("comment.data", "comment.likes"); + + private static final List nameFields = Arrays.asList("firstname", "lastname"); + + private final int RESPONSE_DEFAULT_MAX_SIZE = 200; + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + loadIndex(Index.PHRASE); + loadIndex(Index.GAME_OF_THRONES); + loadIndex(Index.NESTED); + } + + @Override + protected Request getSqlRequest(String request, boolean explain) { + Request sqlRequest = super.getSqlRequest(request, explain); + sqlRequest.addParameter("format", "jdbc"); + + return sqlRequest; + } + + @Ignore("Index type is removed in ES 7+") + @Test + public void wrongIndexType() throws IOException { + String type = "wrongType"; + try { + executeQuery(String.format(Locale.ROOT, "SELECT * FROM %s/%s", + TestsConstants.TEST_INDEX_ACCOUNT, type)); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), + is(String.format(Locale.ROOT, "Index type %s does not exist", type))); + } + } + + @Test + public void selectAll() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * FROM %s", + TestsConstants.TEST_INDEX_ACCOUNT)); + + // This also tests that .keyword fields are ignored when SELECT * is called + assertContainsColumnsInAnyOrder(getSchema(response), allAccountFields); + assertContainsData(getDataRows(response), allAccountFields); + } + + @Test + public void selectNames() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT firstname, lastname FROM %s", + TestsConstants.TEST_INDEX_ACCOUNT)); + + assertContainsColumns(getSchema(response), nameFields); + assertContainsData(getDataRows(response), nameFields); + } + + @Ignore("Semantic analysis takes care of this") + @Test + public void selectWrongField() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT wrongField FROM %s", + TestsConstants.TEST_INDEX_ACCOUNT)); + + assertThat(getSchema(response).length(), equalTo(0)); + + // DataRows object will still get populated with SearchHits but since wrongField is not available in the Map + // each row in the response will be empty + // TODO Perhaps a code change should be made to format logic to ensure a + // 'datarows' length of 0 in response for this case + assertThat(getDataRows(response).length(), equalTo(RESPONSE_DEFAULT_MAX_SIZE)); + } + + @Test + public void selectKeyword() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT firstname.keyword FROM %s", + TestsConstants.TEST_INDEX_ACCOUNT)); + + List fields = Collections.singletonList("firstname.keyword"); + assertContainsColumns(getSchema(response), fields); + + /* + * firstname.keyword will appear in Schema but because there is no 'firstname.keyword' in SearchHits source + * the DataRows will output null. + * + * Looks like x-pack adds this keyword field to "docvalue_fields", this is likely how it ends up in SearchHits + */ + // assertContainsData(getDataRows(protocol), fields); + } + + @Test + public void selectScore() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT _score FROM %s WHERE balance > 30000", + TestsConstants.TEST_INDEX_ACCOUNT)); + + List fields = Collections.singletonList("_score"); + assertContainsColumns(getSchema(response), fields); + assertContainsData(getDataRows(response), fields); + } + + @Test + public void selectAllFromNestedWithoutFieldInFrom() throws IOException { + assertNestedFieldQueryResultContainsColumnsAndData("SELECT * FROM %s", + regularFields, fields("message", "comment")); + } + + @Test + public void selectAllFromNestedWithFieldInFrom() throws IOException { + assertNestedFieldQueryResultContainsColumnsAndData("SELECT * FROM %s e, e.message m", + regularFields, messageFields); + } + + @Test + public void selectAllFromNestedWithMultipleFieldsInFrom() throws IOException { + assertNestedFieldQueryResultContainsColumnsAndData( + "SELECT * FROM %s e, e.message m, e.comment c", + regularFields, messageFields, commentFields); + } + + @Test + public void selectAllNestedFromNestedWithFieldInFrom() throws IOException { + assertNestedFieldQueryResultContainsColumnsAndData("SELECT m.* FROM %s e, e.message m", + messageFields); + } + + @Test + public void selectSpecificRegularFieldAndAllFromNestedWithFieldInFrom() throws IOException { + assertNestedFieldQueryResultContainsColumnsAndData( + "SELECT e.someField, m.* FROM %s e, e.message m", + fields("someField"), messageFields); + } + + /** + * Execute the query against index with nested fields and assert result contains columns and data as expected. + */ + @SafeVarargs + private final void assertNestedFieldQueryResultContainsColumnsAndData(String query, + Set... expectedFieldNames) + throws IOException { + JSONObject response = + executeQuery(String.format(Locale.ROOT, query, TestsConstants.TEST_INDEX_NESTED_TYPE)); + Set allExpectedFieldNames = Stream.of(expectedFieldNames). + flatMap(Set::stream). + collect(toSet()); + + assertContainsColumnsInAnyOrder(getSchema(response), allExpectedFieldNames); + assertContainsData(getDataRows(response), allExpectedFieldNames); + } + + private Set fields(String... fieldNames) { + return Sets.newHashSet(fieldNames); + } + + @Test + public void selectNestedFields() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT nested(message.info), someField FROM %s", + TestsConstants.TEST_INDEX_NESTED_TYPE)); + + List fields = Arrays.asList("message.info", "someField"); + assertContainsColumns(getSchema(response), fields); + assertContainsData(getDataRows(response), fields); + + // The nested test index being used contains 5 entries but one of them has an array of 2 message objects, so + // we check to see if the amount of data rows is 6 since that is the result after flattening + assertThat(getDataRows(response).length(), equalTo(6)); + } + + @Test + public void selectNestedFieldWithWildcard() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT nested(message.*) FROM %s", + TestsConstants.TEST_INDEX_NESTED_TYPE)); + + assertContainsColumnsInAnyOrder(getSchema(response), messageFields); + assertContainsData(getDataRows(response), messageFields); + } + + @Test + public void selectWithWhere() throws IOException { + int balanceToCompare = 30000; + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT balance " + + "FROM %s " + + "WHERE balance > %d", + TestsConstants.TEST_INDEX_ACCOUNT, balanceToCompare)); + + /* + * Previously the DataRows map was used to check specific fields but the JDBC response for "datarows" is a + * JSONArray so keys cannot be used to identify the field. Instead, the expected position will be used to + * retrieve the data from the JSONArray representing each "row". + */ + JSONArray dataRows = getDataRows(response); + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + int balance = row.getInt(0); + + assertThat(balance, greaterThan(balanceToCompare)); + } + } + + @Test + public void groupBySingleField() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * FROM %s GROUP BY age", + TestsConstants.TEST_INDEX_ACCOUNT)); + + List fields = Collections.singletonList("age"); + assertContainsColumns(getSchema(response), fields); + assertContainsData(getDataRows(response), fields); + } + + @Test + public void groupByMultipleFields() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * FROM %s GROUP BY age, balance", + TestsConstants.TEST_INDEX_ACCOUNT)); + + List fields = Arrays.asList("age", "balance"); + assertContainsColumns(getSchema(response), fields); + assertContainsData(getDataRows(response), fields); + } + + @Test + public void testSizeAndTotal() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * " + + "FROM %s " + + "WHERE balance > 30000 " + + "LIMIT 5", + TestsConstants.TEST_INDEX_ACCOUNT)); + + JSONArray dataRows = getDataRows(response); + assertThat(dataRows.length(), equalTo(5)); + + // The value to compare to here was obtained by running the query in the plugin and looking at the SearchHits + int totalHits = response.getInt("total"); + assertThat(totalHits, equalTo(402)); + } + + @Test + public void testSizeWithGroupBy() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * FROM %s GROUP BY age LIMIT 5", + TestsConstants.TEST_INDEX_ACCOUNT)); + + assertThat(getDataRows(response).length(), equalTo(5)); + } + + @Test + public void aggregationFunctionInSelect() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT COUNT(*) FROM %s GROUP BY age", + TestsConstants.TEST_INDEX_ACCOUNT)); + + List fields = Arrays.asList("COUNT(*)"); + assertContainsColumns(getSchema(response), fields); + + JSONArray dataRows = getDataRows(response); + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + long countVal = row.getLong(0); + + assertThat(countVal, greaterThan((long) 0)); + } + } + + @Test + public void aggregationFunctionInSelectCaseCheck() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT count(*) FROM %s GROUP BY age", + TestsConstants.TEST_INDEX_ACCOUNT)); + + List fields = Arrays.asList("COUNT(*)"); + assertContainsColumns(getSchema(response), fields); + + JSONArray dataRows = getDataRows(response); + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + long countVal = row.getLong(0); + + assertThat(countVal, greaterThan((long) 0)); + } + } + + @Test + public void aggregationFunctionInSelectWithAlias() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT COUNT(*) AS total FROM %s GROUP BY age", + TestsConstants.TEST_INDEX_ACCOUNT)); + + List fields = Arrays.asList("total"); + assertContainsColumns(getSchema(response), fields); + + JSONArray dataRows = getDataRows(response); + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + long countVal = row.getLong(0); + + assertThat(countVal, greaterThan((long) 0)); + } + } + + @Test + public void aggregationFunctionInSelectGroupByMultipleFields() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT SUM(age) FROM %s GROUP BY age, state.keyword", + TestsConstants.TEST_INDEX_ACCOUNT)); + + List fields = Arrays.asList("SUM(age)"); + assertContainsColumns(getSchema(response), fields); + assertContainsData(getDataRows(response), fields); + } + + @Test + public void aggregationFunctionInSelectNoGroupBy() throws IOException { + JSONObject response = executeQuery(String.format(Locale.ROOT, "SELECT SUM(age) FROM %s", + TestsConstants.TEST_INDEX_ACCOUNT)); + + String ageSum = "SUM(age)"; + assertContainsColumns(getSchema(response), Collections.singletonList(ageSum)); + + JSONArray dataRows = getDataRows(response); + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + double sumVal = row.getDouble(0); + + assertThat(sumVal, greaterThan((double) 0)); + } + } + + @Test + public void multipleAggregationFunctionsInSelect() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT COUNT(*), AVG(age) FROM %s GROUP BY age", + TestsConstants.TEST_INDEX_ACCOUNT)); + + List fields = Arrays.asList("COUNT(*)", "AVG(age)"); + assertContainsColumns(getSchema(response), fields); + assertContainsData(getDataRows(response), fields); + } + + @Test + public void aggregationFunctionInHaving() throws IOException { + JSONObject response = executeQuery(String.format(Locale.ROOT, + "SELECT gender " + + "FROM %s " + + "GROUP BY gender " + + "HAVING count(*) > 500", + TestsConstants.TEST_INDEX_ACCOUNT)); + + String ageSum = "gender"; + assertContainsColumns(getSchema(response), Collections.singletonList(ageSum)); + + JSONArray dataRows = getDataRows(response); + assertEquals(1, dataRows.length()); + assertEquals("m", dataRows.getJSONArray(0).getString(0)); + } + + /** + * This case doesn't seem to be supported by the plugin at the moment. + * Looks like the painless script of the inner function is put inside the aggregation function but + * this syntax may not be correct since it returns 0 which is the default value (since 0 is returned in + * cases like COUNT(wrongField) as well). + */ +// @Test +// public void nestedAggregationFunctionInSelect() { +// String query = String.format(Locale.ROOT, "SELECT SUM(SQRT(age)) FROM age GROUP BY age", TEST_INDEX_ACCOUNT); +// } + @Test + public void fieldsWithAlias() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT firstname AS first, age AS a FROM %s", + TestsConstants.TEST_INDEX_ACCOUNT)); + + Map aliases = new HashMap<>(); + aliases.put("firstname", "first"); + aliases.put("age", "a"); + + assertContainsAliases(getSchema(response), aliases); + } + + @Test + public void indexWithMissingFields() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT phrase, insert_time2 " + + "FROM %s " + + "WHERE match_phrase(phrase, 'brown fox')", + TestsConstants.TEST_INDEX_PHRASE)); + + JSONArray dataRowEntry = getDataRows(response).getJSONArray(0); + assertThat(dataRowEntry.length(), equalTo(2)); + assertThat(dataRowEntry.get(0), equalTo("brown fox")); + assertThat(dataRowEntry.get(1), + equalTo(JSONObject.NULL)); // TODO See if this null check is failing + } + + @Test + public void joinQuery() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT b1.balance, b1.age, b2.firstname " + + "FROM %s b1 JOIN %s b2 ON b1.age = b2.age", + TestsConstants.TEST_INDEX_ACCOUNT, TestsConstants.TEST_INDEX_ACCOUNT)); + + List fields = Arrays.asList("b1.balance", "b1.age", "b2.firstname"); + assertContainsColumns(getSchema(response), fields); + assertContainsData(getDataRows(response), fields); + } + + @Test + public void joinQueryWithAlias() throws IOException { + JSONObject response = executeQuery(String.format(Locale.ROOT, "SELECT b1.balance AS bal, " + + " b1.age AS age, b2.firstname AS name FROM %s b1 JOIN %s b2 ON b1.age = b2.age", + TestsConstants.TEST_INDEX_ACCOUNT, TestsConstants.TEST_INDEX_ACCOUNT)); + + Map aliases = new HashMap<>(); + aliases.put("b1.balance", "bal"); + aliases.put("b1.age", "age"); + aliases.put("b2.firstname", "name"); + + assertContainsAliases(getSchema(response), aliases); + assertContainsData(getDataRows(response), Arrays.asList("bal", "age", "name")); + } + + @Test + public void joinQueryWithObjectFieldInSelect() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT c.name.firstname, d.name.lastname " + + "FROM %s c JOIN %s d ON d.hname = c.house", + TestsConstants.TEST_INDEX_GAME_OF_THRONES, + TestsConstants.TEST_INDEX_GAME_OF_THRONES)); + + List fields = Arrays.asList("c.name.firstname", "d.name.lastname"); + assertContainsColumns(getSchema(response), fields); + + // d.name.lastname is null here since entries with hname don't have a name.lastname entry, so only length is + // checked + JSONArray dataRows = getDataRows(response); + assertThat(dataRows.length(), greaterThan(0)); + + JSONArray row = dataRows.getJSONArray(0); + assertThat(row.length(), equalTo(fields.size())); + } + + @Test + public void joinQuerySelectOnlyOnOneTable() throws Exception { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT b1.age " + + "FROM %s b1 JOIN %s b2 ON b1.firstname = b2.firstname", + TestsConstants.TEST_INDEX_ACCOUNT, TestsConstants.TEST_INDEX_ACCOUNT)); + + List fields = Collections.singletonList("b1.age"); + assertContainsColumns(getSchema(response), fields); + assertContainsData(getDataRows(response), fields); + } + + @Test + public void fieldOrder() throws IOException { + + final String[] expectedFields = {"age", "firstname", "address", "gender", "email"}; + final Object[] expectedValues = {32, "Amber", "880 Holmes Lane", "M", "amberduke@pyrami.com"}; + + testFieldOrder(expectedFields, expectedValues); + } + + @Test + public void fieldOrderOther() throws IOException { + + final String[] expectedFields = {"email", "firstname", "age", "gender", "address"}; + final Object[] expectedValues = {"amberduke@pyrami.com", "Amber", 32, "M", "880 Holmes Lane"}; + + testFieldOrder(expectedFields, expectedValues); + } + + private void testFieldOrder(final String[] expectedFields, final Object[] expectedValues) + throws IOException { + + final String fields = String.join(", ", expectedFields); + final String query = String.format(Locale.ROOT, "SELECT %s FROM %s " + + "WHERE email='amberduke@pyrami.com'", fields, TestsConstants.TEST_INDEX_ACCOUNT); + final JSONObject result = executeQuery(query); + + for (int i = 0; i < expectedFields.length; ++i) { + + final String fieldName = + (String) result.query(String.format(Locale.ROOT, "/schema/%d/name", i)); + assertThat(fieldName, equalTo(expectedFields[i])); + final Object fieldValue = result.query(String.format(Locale.ROOT, "/datarows/0/%d", i)); + assertThat(fieldValue, equalTo(expectedValues[i])); + } + } + + private JSONArray getSchema(JSONObject jdbcResponse) { + return jdbcResponse.getJSONArray("schema"); + } + + private JSONArray getDataRows(JSONObject jdbcResponse) { + return jdbcResponse.getJSONArray("datarows"); + } + + private void assertContainsColumnsInAnyOrder(JSONArray schema, Set fields) { + + assertThat(schema.length(), equalTo(fields.size())); + + for (int i = 0; i < schema.length(); i++) { + JSONObject column = schema.getJSONObject(i); + String name = column.getString("name"); + + assertTrue(fields.contains(name)); + } + } + + private void assertContainsColumns(JSONArray schema, List fields) { + + assertThat(schema.length(), equalTo(fields.size())); + + for (int i = 0; i < schema.length(); i++) { + JSONObject column = schema.getJSONObject(i); + String name = column.getString("name"); + + assertThat(name, equalTo(fields.get(i))); + } + } + + private void assertContainsAliases(JSONArray schema, Map aliases) { + for (int i = 0; i < schema.length(); i++) { + JSONObject column = schema.getJSONObject(i); + assertTrue(column.has("alias")); + + String name = column.getString("name"); + String alias = column.getString("alias"); + + assertThat(alias, equalTo(aliases.get(name))); + } + } + + private void assertContainsData(JSONArray dataRows, Collection fields) { + assertThat(dataRows.length(), greaterThan(0)); + JSONArray row = dataRows.getJSONArray(0); + + assertThat(row.length(), equalTo(fields.size())); + for (int i = 0; i < row.length(); i++) { + assertThat(row.get(i), not(equalTo(JSONObject.NULL))); + } + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatterIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatterIT.java new file mode 100644 index 0000000000..913cc43d4c --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatterIT.java @@ -0,0 +1,71 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static org.hamcrest.Matchers.equalTo; + +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; +import com.google.common.io.Files; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.junit.Test; + +public class PrettyFormatterIT extends SQLIntegTestCase { + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + } + + @Test + public void assertExplainPrettyFormatted() throws IOException { + String query = StringUtils.format("SELECT firstname FROM %s", TEST_INDEX_ACCOUNT); + + String notPrettyExplainOutputFilePath = TestUtils.getResourceFilePath( + "src/test/resources/expectedOutput/explainIT_format_not_pretty.json"); + String notPrettyExplainOutput = + Files.toString(new File(notPrettyExplainOutputFilePath), StandardCharsets.UTF_8); + + assertThat(executeExplainRequest(query, ""), equalTo(notPrettyExplainOutput)); + assertThat(executeExplainRequest(query, "pretty=false"), equalTo(notPrettyExplainOutput)); + + String prettyExplainOutputFilePath = TestUtils.getResourceFilePath( + "src/test/resources/expectedOutput/explainIT_format_pretty.json"); + String prettyExplainOutput = + Files.toString(new File(prettyExplainOutputFilePath), StandardCharsets.UTF_8); + + assertThat(executeExplainRequest(query, "pretty"), equalTo(prettyExplainOutput)); + assertThat(executeExplainRequest(query, "pretty=true"), equalTo(prettyExplainOutput)); + } + + private String executeExplainRequest(String query, String explainParam) throws IOException { + String endpoint = "/_opendistro/_sql/_explain?" + explainParam; + String request = makeRequest(query); + + Request sqlRequest = new Request("POST", endpoint); + sqlRequest.setJsonEntity(request); + + Response response = client().performRequest(sqlRequest); + String responseString = TestUtils.getResponseBody(response, true); + + return responseString; + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/QueryAnalysisIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/QueryAnalysisIT.java new file mode 100644 index 0000000000..4a5e6d4ccc --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/QueryAnalysisIT.java @@ -0,0 +1,354 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_ENABLED; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_SUGGESTION; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_THRESHOLD; +import static org.elasticsearch.rest.RestStatus.BAD_REQUEST; +import static org.elasticsearch.rest.RestStatus.OK; +import static org.elasticsearch.rest.RestStatus.SERVICE_UNAVAILABLE; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.SemanticAnalysisException; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.syntax.SyntaxAnalysisException; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlFeatureNotImplementedException; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; +import java.io.IOException; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.rest.RestStatus; +import org.junit.Assert; +import org.junit.Test; + +/** + * Integration test for syntax and semantic analysis against query by new ANTLR parser. + */ +public class QueryAnalysisIT extends SQLIntegTestCase { + + @Override + protected void init() throws Exception { + loadIndex(Index.BANK); + } + + @Test + public void missingFromClauseShouldThrowSyntaxException() { + queryShouldThrowSyntaxException("SELECT 1"); + } + + @Test + public void unsupportedOperatorShouldThrowSyntaxException() { + queryShouldThrowSyntaxException( + "SELECT * FROM elasticsearch-sql_test_index_bank WHERE age <=> 1" + ); + } + + @Test + public void unsupportedOperatorShouldSkipAnalysisAndThrowOtherExceptionIfAnalyzerDisabled() { + runWithClusterSetting( + new ClusterSetting("transient", QUERY_ANALYSIS_ENABLED, "false"), + () -> queryShouldThrowException( + "SELECT * FROM elasticsearch-sql_test_index_bank WHERE age <=> 1", + SqlParseException.class + ) + ); + } + + @Test + public void suggestionForWrongFieldNameShouldBeProvidedIfSuggestionEnabled() { + runWithClusterSetting( + new ClusterSetting("transient", QUERY_ANALYSIS_SEMANTIC_SUGGESTION, "true"), + () -> queryShouldThrowSemanticException( + "SELECT * FROM elasticsearch-sql_test_index_bank b WHERE a.balance = 1000", + "Field [a.balance] cannot be found or used here.", + "Did you mean [b.balance]?" + ) + ); + } + + @Test + public void wrongFieldNameShouldPassIfIndexMappingIsVeryLarge() { + runWithClusterSetting( + new ClusterSetting("transient", QUERY_ANALYSIS_SEMANTIC_THRESHOLD, "5"), + () -> queryShouldPassAnalysis( + "SELECT * FROM elasticsearch-sql_test_index_bank WHERE age123 = 1") + ); + } + + /* + @Test + public void useNewAddedFieldShouldPass() throws Exception { + // 1.Make sure new add fields not there originally + String query = "SELECT salary FROM elasticsearch-sql_test_index_bank WHERE education = 'PhD'"; + queryShouldThrowSemanticException(query, "Field [education] cannot be found or used here."); + + // 2.Index an document with fields not present in mapping previously + String docWithNewFields = "{\"account_number\":12345,\"education\":\"PhD\",\"salary\": \"10000\"}"; + IndexResponse resp = client().index(new IndexRequest().index("elasticsearch-sql_test_index_bank"). + source(docWithNewFields, JSON)).get(); + + Assert.assertEquals(RestStatus.CREATED, resp.status()); + + // 3.Same query should pass + executeQuery(query); + } + */ + + @Test + public void nonExistingFieldNameShouldThrowSemanticException() { + queryShouldThrowSemanticException( + "SELECT * FROM elasticsearch-sql_test_index_bank WHERE balance1 = 1000", + "Field [balance1] cannot be found or used here." + //"Did you mean [balance]?" + ); + } + + @Test + public void nonExistingIndexAliasShouldThrowSemanticException() { + queryShouldThrowSemanticException( + "SELECT * FROM elasticsearch-sql_test_index_bank b WHERE a.balance = 1000", + "Field [a.balance] cannot be found or used here." + //"Did you mean [b.balance]?" + ); + } + + @Test + public void indexJoinNonNestedFieldShouldThrowSemanticException() { + queryShouldThrowSemanticException( + "SELECT * FROM elasticsearch-sql_test_index_bank b1, b1.firstname f1", + "Operator [JOIN] cannot work with [INDEX, TEXT]." + ); + } + + @Test + public void scalarFunctionCallWithTypoInNameShouldThrowSemanticException() { + queryShouldThrowSemanticException( + "SELECT * FROM elasticsearch-sql_test_index_bank WHERE ABSa(age) = 1", + "Function [ABSA] cannot be found or used here.", + "Did you mean [ABS]?" + ); + } + + @Test + public void scalarFunctionCallWithWrongTypeArgumentShouldThrowSemanticException() { + queryShouldThrowSemanticException( + "SELECT * FROM elasticsearch-sql_test_index_bank WHERE LOG(lastname) = 1", + "Function [LOG] cannot work with [KEYWORD].", + "Usage: LOG(NUMBER T) -> DOUBLE or LOG(NUMBER T, NUMBER) -> DOUBLE" + ); + } + + @Test + public void aggregateFunctionCallWithWrongNumberOfArgumentShouldThrowSemanticException() { + queryShouldThrowSemanticException( + "SELECT city FROM elasticsearch-sql_test_index_bank GROUP BY city HAVING MAX(age, birthdate) > 1", + "Function [MAX] cannot work with [INTEGER, DATE].", + "Usage: MAX(NUMBER T) -> T" + ); + } + + @Test + public void compareIntegerFieldWithBooleanShouldThrowSemanticException() { + queryShouldThrowSemanticException( + "SELECT * FROM elasticsearch-sql_test_index_bank b WHERE b.age IS FALSE", + "Operator [IS] cannot work with [INTEGER, BOOLEAN].", + "Usage: Please use compatible types from each side." + ); + } + + @Test + public void compareNumberFieldWithStringShouldThrowSemanticException() { + queryShouldThrowSemanticException( + "SELECT * FROM elasticsearch-sql_test_index_bank b WHERE b.age >= 'test'", + "Operator [>=] cannot work with [INTEGER, STRING].", + "Usage: Please use compatible types from each side." + ); + } + + @Test + public void compareLogFunctionCallWithNumberFieldWithStringShouldThrowSemanticException() { + queryShouldThrowSemanticException( + "SELECT * FROM elasticsearch-sql_test_index_bank b WHERE LOG(b.balance) != 'test'", + "Operator [!=] cannot work with [DOUBLE, STRING].", + "Usage: Please use compatible types from each side." + ); + } + + @Test + public void unionNumberFieldWithStringShouldThrowSemanticException() { + queryShouldThrowSemanticException( + "SELECT age FROM elasticsearch-sql_test_index_bank" + + " UNION SELECT address FROM elasticsearch-sql_test_index_bank", + "Operator [UNION] cannot work with [INTEGER, TEXT]." + ); + } + + @Test + public void minusBooleanFieldWithDateShouldThrowSemanticException() { + queryShouldThrowSemanticException( + "SELECT male FROM elasticsearch-sql_test_index_bank" + + " MINUS SELECT birthdate FROM elasticsearch-sql_test_index_bank", + "Operator [MINUS] cannot work with [BOOLEAN, DATE]." + ); + } + + @Test + public void useInClauseWithIncompatibleFieldTypesShouldFail() { + queryShouldThrowSemanticException( + "SELECT * FROM elasticsearch-sql_test_index_bank WHERE male " + + " IN (SELECT 1 FROM elasticsearch-sql_test_index_bank)", + "Operator [IN] cannot work with [BOOLEAN, INTEGER]." + ); + } + + @Test + public void queryWithNestedFunctionShouldFail() { + queryShouldThrowFeatureNotImplementedException( + "SELECT abs(log(balance)) FROM elasticsearch-sql_test_index_bank", + "Nested function calls like [abs(log(balance))] are not supported yet" + ); + } + + @Test + public void nestedFunctionWithMathConstantAsInnerFunctionShouldPass() { + queryShouldPassAnalysis("SELECT log(e()) FROM elasticsearch-sql_test_index_bank"); + } + + @Test + public void aggregateWithFunctionAggregatorShouldFail() { + queryShouldThrowFeatureNotImplementedException( + "SELECT max(log(age)) FROM elasticsearch-sql_test_index_bank", + "Aggregation calls with function aggregator like [max(log(age))] are not supported yet" + ); + } + + @Test + public void queryWithUnsupportedFunctionShouldFail() { + queryShouldThrowFeatureNotImplementedException( + "SELECT balance DIV age FROM elasticsearch-sql_test_index_bank", + "Operator [DIV] is not supported yet" + ); + } + + @Test + public void useNegativeNumberConstantShouldPass() { + queryShouldPassAnalysis( + "SELECT * FROM elasticsearch-sql_test_index_bank " + + "WHERE age > -1 AND balance < -123.456789" + ); + } + + /** + * Run the query with cluster setting changed and cleaned after complete + */ + private void runWithClusterSetting(ClusterSetting setting, Runnable query) { + try { + updateClusterSettings(setting); + query.run(); + } catch (IOException e) { + throw new IllegalStateException( + StringUtils.format("Exception raised when running with cluster setting [%s]", setting)); + } finally { + // Clean up or ES will throw java.lang.AssertionError: test leaves persistent cluster metadata behind + try { + updateClusterSettings(setting.nullify()); + } catch (IOException e) { + // Ignore exception during the cleanup + } + } + } + + private void queryShouldThrowSyntaxException(String query, String... expectedMsgs) { + queryShouldThrowException(query, SyntaxAnalysisException.class, expectedMsgs); + } + + private void queryShouldThrowSemanticException(String query, String... expectedMsgs) { + queryShouldThrowException(query, SemanticAnalysisException.class, expectedMsgs); + } + + private void queryShouldThrowFeatureNotImplementedException(String query, + String... expectedMsgs) { + queryShouldThrowExceptionWithRestStatus(query, SqlFeatureNotImplementedException.class, + SERVICE_UNAVAILABLE, expectedMsgs); + } + + private void queryShouldThrowException(String query, Class exceptionType, + String... expectedMsgs) { + queryShouldThrowExceptionWithRestStatus(query, exceptionType, BAD_REQUEST, expectedMsgs); + } + + private void queryShouldThrowExceptionWithRestStatus(String query, Class exceptionType, + RestStatus status, + String... expectedMsgs) { + try { + executeQuery(query); + Assert.fail("Expected ResponseException, but none was thrown for query: " + query); + } catch (ResponseException e) { + ResponseAssertion assertion = new ResponseAssertion(e.getResponse()); + assertion.assertStatusEqualTo(status.getStatus()); + assertion.assertBodyContains("\"type\": \"" + exceptionType.getSimpleName() + "\""); + for (String msg : expectedMsgs) { + assertion.assertBodyContains(msg); + } + } catch (IOException e) { + throw new IllegalStateException( + "Unexpected IOException raised rather than expected AnalysisException for query: " + + query); + } + } + + private void queryShouldPassAnalysis(String query) { + String endpoint = "/_opendistro/_sql?"; + String requestBody = makeRequest(query); + Request sqlRequest = new Request("POST", endpoint); + sqlRequest.setJsonEntity(requestBody); + + try { + Response response = client().performRequest(sqlRequest); + ResponseAssertion assertion = new ResponseAssertion(response); + assertion.assertStatusEqualTo(OK.getStatus()); + } catch (IOException e) { + throw new IllegalStateException("Unexpected IOException raised for query: " + query); + } + } + + private static class ResponseAssertion { + private final Response response; + private final String body; + + ResponseAssertion(Response response) { + this.response = response; + try { + this.body = TestUtils.getResponseBody(response); + } catch (IOException e) { + throw new IllegalStateException("Unexpected IOException raised when reading response body"); + } + } + + void assertStatusEqualTo(int expectedStatus) { + assertThat(response.getStatusLine().getStatusCode(), equalTo(expectedStatus)); + } + + void assertBodyContains(String content) { + assertThat(body, containsString(content)); + } + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/QueryFunctionsIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/QueryFunctionsIT.java new file mode 100644 index 0000000000..463bf97041 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/QueryFunctionsIT.java @@ -0,0 +1,305 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_TYPE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_PHRASE; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.everyItem; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.startsWith; + +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.search.SearchHit; +import org.hamcrest.BaseMatcher; +import org.hamcrest.Description; +import org.hamcrest.FeatureMatcher; +import org.hamcrest.Matcher; +import org.json.JSONObject; +import org.junit.Test; + +public class QueryFunctionsIT extends SQLIntegTestCase { + + private static final String SELECT_ALL = "SELECT *"; + private static final String FROM_ACCOUNTS = "FROM " + TEST_INDEX_ACCOUNT; + private static final String FROM_NESTED = "FROM " + TEST_INDEX_NESTED_TYPE; + private static final String FROM_PHRASE = "FROM " + TEST_INDEX_PHRASE; + + /** + * TODO Looks like Math/Date Functions test all use the same query() and execute() functions + * TODO execute/featureValueOf/hits functions are the same as used in NestedFieldQueryIT, should refactor into util + */ + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + loadIndex(Index.NESTED); + loadIndex(Index.PHRASE); + } + + @Test + public void query() throws IOException { + assertThat( + query( + "SELECT state", + FROM_ACCOUNTS, + "WHERE QUERY('CA')" + ), + hits( + hasValueForFields("CA", "state") + ) + ); + } + + @Test + public void matchQueryRegularField() throws IOException { + assertThat( + query( + "SELECT firstname", + FROM_ACCOUNTS, + "WHERE MATCH_QUERY(firstname, 'Ayers')" + ), + hits( + hasValueForFields("Ayers", "firstname") + ) + ); + } + + @Test + public void matchQueryNestedField() throws IOException { + SearchHit[] hits = + query("SELECT comment.data", FROM_NESTED, "WHERE MATCH_QUERY(NESTED(comment.data), 'aa')") + .getHits().getHits(); + Map source = hits[0].getSourceAsMap(); + // SearchHits innerHits = hits[0].getInnerHits().get("comment"); + assertThat( + query( + "SELECT comment.data", + FROM_NESTED, + "WHERE MATCH_QUERY(NESTED(comment.data), 'aa')" + ), + hits( + anyOf(hasNestedField("comment", "data", "aa"), + hasNestedArrayField("comment", "data", "aa")) + ) + ); + } + + @Test + public void scoreQuery() throws IOException { + assertThat( + query( + "SELECT firstname", + FROM_ACCOUNTS, + "WHERE SCORE(MATCH_QUERY(firstname, 'Ayers'), 10)" + ), + hits( + hasValueForFields("Ayers", "firstname") + ) + ); + } + + @Test + public void scoreQueryWithNestedField() throws IOException { + assertThat( + query( + "SELECT comment.data", + FROM_NESTED, + "WHERE SCORE(MATCH_QUERY(NESTED(comment.data), 'ab'), 10)" + ), + hits( + //hasValueForFields("ab", "comment.data") + hasNestedField("comment", + "data", "ab") + ) + ); + } + + @Test + public void wildcardQuery() throws IOException { + assertThat( + query( + "SELECT city", + FROM_ACCOUNTS, + "WHERE WILDCARD_QUERY(city.keyword, 'B*')" + ), + hits( + hasFieldWithPrefix("city", "B") + ) + ); + } + + @Test + public void matchPhraseQuery() throws IOException { + assertThat( + query( + "SELECT phrase", + FROM_PHRASE, + "WHERE MATCH_PHRASE(phrase, 'brown fox')" + ), + hits( + hasValueForFields("brown fox", "phrase") + ) + ); + } + + @Test + public void multiMatchQuerySingleField() throws IOException { + assertThat( + query( + "SELECT firstname", + FROM_ACCOUNTS, + "WHERE MULTI_MATCH('query'='Ayers', 'fields'='firstname')" + ), + hits( + hasValueForFields("Ayers", "firstname") + ) + ); + } + + @Test + public void multiMatchQueryWildcardField() throws IOException { + assertThat( + query( + "SELECT firstname, lastname", + FROM_ACCOUNTS, + "WHERE MULTI_MATCH('query'='Bradshaw', 'fields'='*name')" + ), + hits( + hasValueForFields("Bradshaw", "firstname", "lastname") + ) + ); + } + + @Test + public void numberLiteralInSelectField() { + assertTrue( + executeQuery(StringUtils.format("SELECT 234234 AS number from %s", TEST_INDEX_ACCOUNT), + "jdbc") + .contains("234234") + ); + + assertTrue( + executeQuery(StringUtils.format("SELECT 2.34234 AS number FROM %s", TEST_INDEX_ACCOUNT), + "jdbc") + .contains("2.34234") + ); + } + + private final Matcher hits(Matcher subMatcher) { + return featureValueOf("hits", everyItem(subMatcher), + resp -> Arrays.asList(resp.getHits().getHits())); + } + + private FeatureMatcher featureValueOf(String name, Matcher subMatcher, + Function getter) { + return new FeatureMatcher(subMatcher, name, name) { + @Override + protected U featureValueOf(T actual) { + return getter.apply(actual); + } + }; + } + + /** + * Create Matchers for each field and its value + * Only one of the Matchers need to match (per hit) + *

+ * Ex. If a query with wildcard field is made: + * multi_match(query="Ayers", fields="*name") + *

+ * Then the value "Ayers" can be found in either the firstname or lastname field. Only one of these fields + * need to satisfy the query value to be evaluated as correct expected output. + * + * @param value The value to match for a field in the sourceMap + * @param fields A list of fields to match + */ + @SafeVarargs + private final Matcher hasValueForFields(String value, String... fields) { + return anyOf( + Arrays.asList(fields). + stream(). + map(field -> kv(field, is(value))). + collect(Collectors.toList())); + } + + private final Matcher hasFieldWithPrefix(String field, String prefix) { + return featureValueOf(field, startsWith(prefix), + hit -> (String) hit.getSourceAsMap().get(field)); + } + + private final Matcher hasNestedField(String path, String field, String value) { + return featureValueOf(field, is(value), + hit -> ((HashMap) hit.getSourceAsMap().get(path)).get(field)); + } + + private final Matcher hasNestedArrayField(String path, String field, String value) { + + return new BaseMatcher() { + @Override + public void describeTo(Description description) { + + } + + @Override + public boolean matches(Object item) { + + SearchHit hit = (SearchHit) item; + List elements = + (List) ((HashMap) hit.getSourceAsMap().get(path)).get(field); + return elements.contains(value); + } + }; + } + + private Matcher kv(String key, Matcher valMatcher) { + return featureValueOf(key, valMatcher, hit -> hit.getSourceAsMap().get(key)); + } + + /*********************************************************** + Query Utility to Fetch Response for SQL + ***********************************************************/ + + private SearchResponse query(String select, String from, String... statements) + throws IOException { + return execute(select + " " + from + " " + String.join(" ", statements)); + } + + private SearchResponse execute(String sql) throws IOException { + final JSONObject jsonObject = executeQuery(sql); + + final XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser( + NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, + jsonObject.toString()); + return SearchResponse.fromXContent(parser); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/QueryIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/QueryIT.java new file mode 100644 index 0000000000..f4a4c36a16 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/QueryIT.java @@ -0,0 +1,1812 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_GAME_OF_THRONES; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_TYPE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ONLINE; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.isOneOf; +import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.not; + +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Locale; +import java.util.Set; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.rest.RestStatus; +import org.joda.time.DateTime; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + +public class QueryIT extends SQLIntegTestCase { + + /** + * Currently commenting out tests related to JoinType index since there is an issue with mapping. + *

+ * Also ignoring the following tests as they are failing, will require investigation: + * - idsQuerySubQueryIds + * - escapedCharactersCheck + * - fieldCollapsingTest + * - idsQueryOneId + * - idsQueryMultipleId + * - multipleIndicesOneNotExistWithoutHint + *

+ * The following tests are being ignored because subquery is still running in ES transport thread: + * - twoSubQueriesTest() + * - inTermsSubQueryTest() + */ + + final static int BANK_INDEX_MALE_TRUE = 4; + final static int BANK_INDEX_MALE_FALSE = 3; + + @Override + protected void init() throws Exception { + loadIndex(Index.ONLINE); + loadIndex(Index.ACCOUNT); + loadIndex(Index.PHRASE); + loadIndex(Index.DOG); + loadIndex(Index.PEOPLE); + loadIndex(Index.GAME_OF_THRONES); + loadIndex(Index.ODBC); + loadIndex(Index.LOCATION); + loadIndex(Index.NESTED); + // TODO Remove comment after issue with loading join type is resolved + // loadIndex(Index.JOIN); + loadIndex(Index.BANK); + loadIndex(Index.BANK_TWO); + loadIndex(Index.BANK_WITH_NULL_VALUES); + } + + @Test + public void searchTypeTest() throws IOException { + JSONObject response = executeQuery(String.format(Locale.ROOT, "SELECT * FROM %s LIMIT 1000", + TestsConstants.TEST_INDEX_PHRASE)); + Assert.assertTrue(response.has("hits")); + Assert.assertEquals(6, getTotalHits(response)); + } + + @Test + public void multipleFromTest() throws IOException { + JSONObject response = executeQuery(String.format(Locale.ROOT, + "SELECT * FROM %s, %s LIMIT 2000", + TestsConstants.TEST_INDEX_BANK, TestsConstants.TEST_INDEX_BANK_TWO)); + Assert.assertTrue(response.has("hits")); + Assert.assertEquals(14, getTotalHits(response)); + } + + @Test + public void selectAllWithFieldReturnsAll() throws IOException { + JSONObject response = executeQuery(StringUtils.format( + "SELECT *, age " + + "FROM %s " + + "LIMIT 5", + TestsConstants.TEST_INDEX_BANK + )); + + checkSelectAllAndFieldResponseSize(response); + } + + @Test + public void selectAllWithFieldReverseOrder() throws IOException { + JSONObject response = executeQuery(StringUtils.format( + "SELECT *, age " + + "FROM %s " + + "LIMIT 5", + TestsConstants.TEST_INDEX_BANK + )); + + checkSelectAllAndFieldResponseSize(response); + } + + @Test + public void selectAllWithMultipleFields() throws IOException { + JSONObject response = executeQuery(StringUtils.format( + "SELECT *, age, address " + + "FROM %s " + + "LIMIT 5", + TestsConstants.TEST_INDEX_BANK + )); + + checkSelectAllAndFieldResponseSize(response); + } + + @Test + public void selectAllWithFieldAndOrderBy() throws IOException { + JSONObject response = executeQuery(StringUtils.format( + "SELECT *, age " + + "FROM %s " + + "ORDER BY age " + + "LIMIT 5", + TestsConstants.TEST_INDEX_BANK + )); + + checkSelectAllAndFieldResponseSize(response); + } + + @Test + public void selectAllWithFieldAndGroupBy() throws IOException { + JSONObject response = executeQuery(StringUtils.format( + "SELECT *, age " + + "FROM %s " + + "GROUP BY age " + + "LIMIT 10", + TestsConstants.TEST_INDEX_BANK + )); + + checkSelectAllAndFieldAggregationResponseSize(response, "age"); + } + + @Test + public void selectAllWithFieldAndGroupByReverseOrder() throws IOException { + JSONObject response = executeQuery(StringUtils.format( + "SELECT *, age " + + "FROM %s " + + "GROUP BY age " + + "LIMIT 10", + TestsConstants.TEST_INDEX_BANK + )); + + checkSelectAllAndFieldAggregationResponseSize(response, "age"); + } + + @Test + public void selectFieldWithAliasAndGroupBy() { + String response = + executeQuery("SELECT lastname AS name FROM " + TEST_INDEX_ACCOUNT + " GROUP BY name", + "jdbc"); + assertThat(response, containsString("\"alias\": \"name\"")); + } + + public void indexWithWildcardTest() throws IOException { + JSONObject response = executeQuery(String.format(Locale.ROOT, "SELECT * FROM %s* LIMIT 1000", + TestsConstants.TEST_INDEX_BANK)); + Assert.assertTrue(response.has("hits")); + assertThat(getTotalHits(response), greaterThan(0)); + } + + @Test + public void selectSpecificFields() throws IOException { + String[] arr = new String[] {"age", "account_number"}; + Set expectedSource = new HashSet<>(Arrays.asList(arr)); + + JSONObject response = + executeQuery(String.format(Locale.ROOT, "SELECT age, account_number FROM %s", + TEST_INDEX_ACCOUNT)); + assertResponseForSelectSpecificFields(response, expectedSource); + } + + @Test + public void selectSpecificFieldsUsingTableAlias() throws IOException { + String[] arr = new String[] {"age", "account_number"}; + Set expectedSource = new HashSet<>(Arrays.asList(arr)); + + JSONObject response = + executeQuery(String.format(Locale.ROOT, "SELECT a.age, a.account_number FROM %s a", + TEST_INDEX_ACCOUNT)); + assertResponseForSelectSpecificFields(response, expectedSource); + } + + @Test + public void selectSpecificFieldsUsingTableNamePrefix() throws IOException { + String[] arr = new String[] {"age", "account_number"}; + Set expectedSource = new HashSet<>(Arrays.asList(arr)); + + JSONObject response = executeQuery(String.format(Locale.ROOT, + "SELECT elasticsearch-sql_test_index_account.age, elasticsearch-sql_test_index_account.account_number" + + " FROM %s", + TEST_INDEX_ACCOUNT)); + assertResponseForSelectSpecificFields(response, expectedSource); + } + + private void assertResponseForSelectSpecificFields(JSONObject response, + Set expectedSource) { + JSONArray hits = getHits(response); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + Assert.assertEquals(expectedSource, getSource(hit).keySet()); + } + } + + @Ignore("Will fix this in issue https://github.com/opendistro-for-elasticsearch/sql/issues/121") + @Test + public void selectFieldWithSpace() throws IOException { + String[] arr = new String[] {"test field"}; + Set expectedSource = new HashSet<>(Arrays.asList(arr)); + + JSONObject response = executeQuery(String.format(Locale.ROOT, "SELECT ['test field'] FROM %s " + + "WHERE ['test field'] IS NOT null", + TestsConstants.TEST_INDEX_PHRASE)); + + JSONArray hits = getHits(response); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + Assert.assertEquals(expectedSource, getSource(hit).keySet()); + } + } + + @Ignore("field aliases are not supported currently") + // it might be possible to change field names after the query already executed. + @Test + public void selectAliases() throws IOException { + + String[] arr = new String[] {"myage", "myaccount_number"}; + Set expectedSource = new HashSet<>(Arrays.asList(arr)); + + JSONObject result = executeQuery(String.format(Locale.ROOT, + "SELECT age AS myage, account_number AS myaccount_number FROM %s", TEST_INDEX_ACCOUNT)); + JSONArray hits = getHits(result); + hits.forEach(hitObj -> { + JSONObject hit = (JSONObject) hitObj; + Assert.assertEquals(expectedSource, hit.getJSONObject("_source").keySet()); + }); + } + + @Test + public void useTableAliasInWhereClauseTest() throws IOException { + JSONObject response = executeQuery(String.format(Locale.ROOT, + "SELECT * FROM %s a WHERE a.city = 'Nogal' LIMIT 1000", TEST_INDEX_ACCOUNT)); + + JSONArray hits = getHits(response); + Assert.assertEquals(1, getTotalHits(response)); + Assert.assertEquals("Nogal", getSource(hits.getJSONObject(0)).get("city")); + } + + @Test + public void notUseTableAliasInWhereClauseTest() throws IOException { + JSONObject response = executeQuery(String.format(Locale.ROOT, + "SELECT * FROM %s a WHERE city = 'Nogal' LIMIT 1000", TEST_INDEX_ACCOUNT)); + + JSONArray hits = getHits(response); + Assert.assertEquals(1, getTotalHits(response)); + Assert.assertEquals("Nogal", getSource(hits.getJSONObject(0)).get("city")); + } + + @Test + public void useTableNamePrefixInWhereClauseTest() throws IOException { + JSONObject response = executeQuery(String.format(Locale.ROOT, + "SELECT * FROM %s WHERE elasticsearch-sql_test_index_account.city = 'Nogal' LIMIT 1000", + TEST_INDEX_ACCOUNT + )); + + JSONArray hits = getHits(response); + Assert.assertEquals(1, getTotalHits(response)); + Assert.assertEquals("Nogal", getSource(hits.getJSONObject(0)).get("city")); + } + + @Test + public void equalityTest() throws IOException { + JSONObject response = executeQuery(String.format(Locale.ROOT, + "SELECT * FROM %s WHERE city = 'Nogal' LIMIT 1000", TEST_INDEX_ACCOUNT)); + + JSONArray hits = getHits(response); + Assert.assertEquals(1, getTotalHits(response)); + Assert.assertEquals("Nogal", getSource(hits.getJSONObject(0)).get("city")); + } + + @Test + public void equalityTestPhrase() throws IOException { + JSONObject response = executeQuery(String.format(Locale.ROOT, "SELECT * FROM %s WHERE " + + "match_phrase(phrase, 'quick fox here') LIMIT 1000", + TestsConstants.TEST_INDEX_PHRASE)); + + JSONArray hits = getHits(response); + Assert.assertEquals(1, getTotalHits(response)); + Assert.assertEquals("quick fox here", getSource(hits.getJSONObject(0)).getString("phrase")); + } + + @Test + public void greaterThanTest() throws IOException { + int someAge = 25; + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * FROM %s WHERE age > %s LIMIT 1000", + TestsConstants.TEST_INDEX_PEOPLE, + someAge)); + + JSONArray hits = getHits(response); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + int age = getSource(hit).getInt("age"); + assertThat(age, greaterThan(someAge)); + } + } + + @Test + public void greaterThanOrEqualTest() throws IOException { + int someAge = 25; + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * FROM %s WHERE age >= %s LIMIT 1000", + TEST_INDEX_ACCOUNT, + someAge)); + + boolean isEqualFound = false; + JSONArray hits = getHits(response); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + int age = getSource(hit).getInt("age"); + assertThat(age, greaterThanOrEqualTo(someAge)); + + if (age == someAge) { + isEqualFound = true; + } + } + + Assert.assertTrue( + String.format(Locale.ROOT, "At least one of the documents need to contains age equal to %s", + someAge), + isEqualFound); + } + + @Test + public void lessThanTest() throws IOException { + int someAge = 25; + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * FROM %s WHERE age < %s LIMIT 1000", + TestsConstants.TEST_INDEX_PEOPLE, + someAge)); + + JSONArray hits = getHits(response); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + int age = getSource(hit).getInt("age"); + assertThat(age, lessThan(someAge)); + } + } + + @Test + public void lessThanOrEqualTest() throws IOException { + int someAge = 25; + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * FROM %s WHERE age <= %s LIMIT 1000", + TEST_INDEX_ACCOUNT, + someAge)); + + boolean isEqualFound = false; + JSONArray hits = getHits(response); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + int age = getSource(hit).getInt("age"); + assertThat(age, lessThanOrEqualTo(someAge)); + + if (age == someAge) { + isEqualFound = true; + } + } + + Assert.assertTrue( + String.format(Locale.ROOT, "At least one of the documents need to contains age equal to %s", + someAge), + isEqualFound); + } + + @Test + public void orTest() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * " + + "FROM %s " + + "WHERE match_phrase(gender, 'F') OR match_phrase(gender, 'M') " + + "LIMIT 1000", TEST_INDEX_ACCOUNT)); + Assert.assertEquals(1000, getTotalHits(response)); + } + + @Test + public void andTest() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * FROM %s WHERE age=32 AND gender='M' LIMIT 1000", + TestsConstants.TEST_INDEX_PEOPLE)); + + JSONArray hits = getHits(response); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + Assert.assertEquals(32, getSource(hit).getInt("age")); + Assert.assertEquals("M", getSource(hit).getString("gender")); + } + } + + @Test + public void likeTest() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * FROM %s WHERE firstname LIKE 'amb%%' LIMIT 1000", + TEST_INDEX_ACCOUNT)); + + JSONArray hits = getHits(response); + Assert.assertEquals(1, getTotalHits(response)); + Assert.assertEquals("Amber", getSource(hits.getJSONObject(0)).getString("firstname")); + } + + @Test + public void notLikeTest() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * FROM %s WHERE firstname NOT LIKE 'amb%%'", + TEST_INDEX_ACCOUNT)); + + JSONArray hits = getHits(response); + Assert.assertNotEquals(0, getTotalHits(response)); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + Assert.assertFalse(getSource(hit).getString("firstname").toLowerCase().startsWith("amb")); + } + } + + @Test + public void regexQueryTest() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * " + + "FROM %s " + + "WHERE dog_name = REGEXP_QUERY('sn.*', 'INTERSECTION|COMPLEMENT|EMPTY', 10000)", + TestsConstants.TEST_INDEX_DOG)); + + JSONArray hits = getHits(response); + Assert.assertEquals(1, hits.length()); + + JSONObject hitSource = getSource(hits.getJSONObject(0)); + Assert.assertEquals("snoopy", hitSource.getString("dog_name")); + Assert.assertEquals("Hattie", hitSource.getString("holdersName")); + Assert.assertEquals(4, hitSource.getInt("age")); + } + + @Test + public void doubleNotTest() throws IOException { + JSONObject response1 = executeQuery( + String.format(Locale.ROOT, + "SELECT * FROM %s WHERE NOT gender LIKE 'm' AND NOT gender LIKE 'f'", + TEST_INDEX_ACCOUNT)); + Assert.assertEquals(0, getTotalHits(response1)); + + JSONObject response2 = executeQuery( + String.format(Locale.ROOT, + "SELECT * FROM %s WHERE NOT gender LIKE 'm' AND gender NOT LIKE 'f'", + TEST_INDEX_ACCOUNT)); + Assert.assertEquals(0, getTotalHits(response2)); + + JSONObject response3 = executeQuery( + String.format(Locale.ROOT, + "SELECT * FROM %s WHERE gender NOT LIKE 'm' AND gender NOT LIKE 'f'", + TEST_INDEX_ACCOUNT)); + Assert.assertEquals(0, getTotalHits(response3)); + + JSONObject response4 = executeQuery( + String.format(Locale.ROOT, + "SELECT * FROM %s WHERE gender LIKE 'm' AND NOT gender LIKE 'f'", + TEST_INDEX_ACCOUNT)); + // Assert there are results and they all have gender 'm' + Assert.assertNotEquals(0, getTotalHits(response4)); + JSONArray hits = getHits(response4); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + Assert.assertEquals("m", getSource(hit).getString("gender").toLowerCase()); + } + + JSONObject response5 = executeQuery( + String.format(Locale.ROOT, "SELECT * FROM %s WHERE NOT (gender = 'm' OR gender = 'f')", + TEST_INDEX_ACCOUNT)); + Assert.assertEquals(0, getTotalHits(response5)); + } + + @Test + public void limitTest() throws IOException { + JSONObject response = executeQuery(String.format(Locale.ROOT, "SELECT * FROM %s LIMIT 30", + TEST_INDEX_ACCOUNT)); + + JSONArray hits = getHits(response); + Assert.assertEquals(30, hits.length()); + } + + @Test + public void betweenTest() throws IOException { + int min = 27; + int max = 30; + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * FROM %s WHERE age BETWEEN %s AND %s LIMIT 1000", + TestsConstants.TEST_INDEX_PEOPLE, min, max)); + + JSONArray hits = getHits(response); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + int age = getSource(hit).getInt("age"); + assertThat(age, allOf(greaterThanOrEqualTo(min), lessThanOrEqualTo(max))); + } + } + + // TODO When using NOT BETWEEN on fields, documents not containing the field + // are returned as well. This may be incorrect behavior. + @Test + public void notBetweenTest() throws IOException { + int min = 20; + int max = 37; + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * FROM %s WHERE age NOT BETWEEN %s AND %s LIMIT 1000", + TestsConstants.TEST_INDEX_PEOPLE, min, max)); + + JSONArray hits = getHits(response); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + JSONObject source = getSource(hit); + + // Ignore documents which do not contain the age field + if (source.has("age")) { + int age = source.getInt("age"); + assertThat(age, not(allOf(greaterThanOrEqualTo(min), lessThanOrEqualTo(max)))); + } + } + } + + @Ignore("Semantic analysis failed because 'age' doesn't exist.") + @Test + public void inTest() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT age FROM %s WHERE age IN (20, 22) LIMIT 1000", + TestsConstants.TEST_INDEX_PHRASE)); + + JSONArray hits = getHits(response); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + int age = getSource(hit).getInt("age"); + assertThat(age, isOneOf(20, 22)); + } + } + + @Test + public void inTestWithStrings() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, + "SELECT phrase FROM %s WHERE phrase IN ('quick', 'fox') LIMIT 1000", + TestsConstants.TEST_INDEX_PHRASE)); + + JSONArray hits = getHits(response); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + String phrase = getSource(hit).getString("phrase"); + assertThat(phrase, isOneOf("quick fox here", "fox brown", "quick fox", "brown fox")); + } + } + + @Test + public void inTermsTestWithIdentifiersTreatedLikeStrings() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT name " + + "FROM %s " + + "WHERE name.firstname = IN_TERMS('daenerys','eddard') " + + "LIMIT 1000", + TestsConstants.TEST_INDEX_GAME_OF_THRONES)); + + JSONArray hits = getHits(response); + Assert.assertEquals(2, getTotalHits(response)); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + String firstname = ((JSONObject) getSource(hit).get("name")).getString("firstname"); + assertThat(firstname, isOneOf("Daenerys", "Eddard")); + } + } + + @Test + public void inTermsTestWithStrings() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT name " + + "FROM %s " + + "WHERE name.firstname = IN_TERMS('daenerys','eddard') " + + "LIMIT 1000", + TestsConstants.TEST_INDEX_GAME_OF_THRONES)); + + JSONArray hits = getHits(response); + Assert.assertEquals(2, getTotalHits(response)); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + String firstname = ((JSONObject) getSource(hit).get("name")).getString("firstname"); + assertThat(firstname, isOneOf("Daenerys", "Eddard")); + } + } + + @Test + public void inTermsWithNumbers() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT name " + + "FROM %s " + + "WHERE name.ofHisName = IN_TERMS(4,2) " + + "LIMIT 1000", + TestsConstants.TEST_INDEX_GAME_OF_THRONES)); + + JSONArray hits = getHits(response); + Assert.assertEquals(1, getTotalHits(response)); + + JSONObject hit = hits.getJSONObject(0); + String firstname = ((JSONObject) getSource(hit).get("name")).getString("firstname"); + Assert.assertEquals("Brandon", firstname); + } + + @Test + public void termQueryWithNumber() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, + "SELECT name FROM %s WHERE name.ofHisName = term(4) LIMIT 1000", + TestsConstants.TEST_INDEX_GAME_OF_THRONES)); + + JSONArray hits = getHits(response); + Assert.assertEquals(1, getTotalHits(response)); + + JSONObject hit = hits.getJSONObject(0); + String firstname = ((JSONObject) getSource(hit).get("name")).getString("firstname"); + Assert.assertEquals("Brandon", firstname); + } + + @Test + public void termQueryWithStringIdentifier() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT name " + + "FROM %s " + + "WHERE name.firstname = term('brandon') " + + "LIMIT 1000", + TestsConstants.TEST_INDEX_GAME_OF_THRONES)); + + JSONArray hits = getHits(response); + Assert.assertEquals(1, getTotalHits(response)); + + JSONObject hit = hits.getJSONObject(0); + String firstname = ((JSONObject) getSource(hit).get("name")).getString("firstname"); + Assert.assertEquals("Brandon", firstname); + } + + @Test + public void termQueryWithStringLiteral() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT name " + + "FROM %s " + + "WHERE name.firstname = term('brandon') " + + "LIMIT 1000", + TestsConstants.TEST_INDEX_GAME_OF_THRONES)); + + JSONArray hits = getHits(response); + Assert.assertEquals(1, getTotalHits(response)); + + JSONObject hit = hits.getJSONObject(0); + String firstname = ((JSONObject) getSource(hit).get("name")).getString("firstname"); + Assert.assertEquals("Brandon", firstname); + } + + // TODO When using NOT IN on fields, documents not containing the field + // are returned as well. This may be incorrect behavior. + @Test + public void notInTest() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT age FROM %s WHERE age NOT IN (20, 22) LIMIT 1000", + TestsConstants.TEST_INDEX_PEOPLE)); + + JSONArray hits = getHits(response); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + JSONObject source = getSource(hit); + + // Ignore documents which do not contain the age field + if (source.has("age")) { + int age = source.getInt("age"); + assertThat(age, not(isOneOf(20, 22))); + } + } + } + + @Test + public void dateSearch() throws IOException { + DateTimeFormatter formatter = DateTimeFormat.forPattern(TestsConstants.DATE_FORMAT); + DateTime dateToCompare = new DateTime(2014, 8, 18, 0, 0, 0); + + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT insert_time FROM %s WHERE insert_time < '2014-08-18'", + TestsConstants.TEST_INDEX_ONLINE)); + JSONArray hits = getHits(response); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + JSONObject source = getSource(hit); + DateTime insertTime = formatter.parseDateTime(source.getString("insert_time")); + + String errorMessage = + String.format(Locale.ROOT, "insert_time must be before 2014-08-18. Found: %s", + insertTime); + Assert.assertTrue(errorMessage, insertTime.isBefore(dateToCompare)); + } + } + + @Test + public void dateSearchBraces() throws IOException { + DateTimeFormatter formatter = DateTimeFormat.forPattern(TestsConstants.TS_DATE_FORMAT); + DateTime dateToCompare = new DateTime(2015, 3, 15, 0, 0, 0); + + JSONObject response = executeQuery( + String.format(Locale.ROOT, + "SELECT odbc_time FROM %s/odbc WHERE odbc_time < {ts '2015-03-15 00:00:00.000'}", + TestsConstants.TEST_INDEX_ODBC)); + JSONArray hits = getHits(response); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + JSONObject source = getSource(hit); + String insertTimeStr = source.getString("odbc_time"); + insertTimeStr = insertTimeStr.replace("{ts '", "").replace("'}", ""); + + DateTime insertTime = formatter.parseDateTime(insertTimeStr); + String errorMessage = + String.format(Locale.ROOT, "insert_time must be before 2015-03-15. Found: %s", + insertTime); + Assert.assertTrue(errorMessage, insertTime.isBefore(dateToCompare)); + } + } + + @Test + public void dateBetweenSearch() throws IOException { + DateTimeFormatter formatter = DateTimeFormat.forPattern(TestsConstants.DATE_FORMAT); + + DateTime dateLimit1 = new DateTime(2014, 8, 18, 0, 0, 0); + DateTime dateLimit2 = new DateTime(2014, 8, 21, 0, 0, 0); + + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT insert_time " + + "FROM %s " + + "WHERE insert_time BETWEEN '2014-08-18' AND '2014-08-21' " + + "LIMIT 3", + TestsConstants.TEST_INDEX_ONLINE)); + JSONArray hits = getHits(response); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + JSONObject source = getSource(hit); + DateTime insertTime = formatter.parseDateTime(source.getString("insert_time")); + + boolean isBetween = (insertTime.isAfter(dateLimit1) || insertTime.isEqual(dateLimit1)) && + (insertTime.isBefore(dateLimit2) || insertTime.isEqual(dateLimit2)); + + Assert.assertTrue("insert_time must be between 2014-08-18 and 2014-08-21", isBetween); + } + } + + @Test + public void missFilterSearch() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * FROM %s WHERE insert_time2 IS missing", + TestsConstants.TEST_INDEX_PHRASE)); + + JSONArray hits = getHits(response); + Assert.assertEquals(4, getTotalHits(response)); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + JSONObject source = getSource(hit); + + Assert.assertFalse(source.has("insert_time2")); + } + } + + @Test + public void notMissFilterSearch() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * FROM %s WHERE insert_time2 IS NOT missing", + TestsConstants.TEST_INDEX_PHRASE)); + + JSONArray hits = getHits(response); + Assert.assertEquals(2, getTotalHits(response)); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + JSONObject source = getSource(hit); + + Assert.assertTrue(source.has("insert_time2")); + } + } + + @Test + public void complexConditionQuery() throws IOException { + String errorMessage = "Result does not exist to the condition " + + "(gender='m' AND (age> 25 OR account_number>5)) OR (gender='f' AND (age>30 OR account_number < 8)"; + + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * " + + "FROM %s " + + "WHERE (gender='m' AND (age> 25 OR account_number>5)) " + + "OR (gender='f' AND (age>30 OR account_number < 8))", + TEST_INDEX_ACCOUNT)); + + JSONArray hits = getHits(response); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + JSONObject source = getSource(hit); + + String gender = source.getString("gender").toLowerCase(); + int age = source.getInt("age"); + int accountNumber = source.getInt("account_number"); + + Assert.assertTrue(errorMessage, + (gender.equals("m") && (age > 25 || accountNumber > 5)) + || (gender.equals("f") && (age > 30 || accountNumber < 8))); + } + } + + @Test + public void complexNotConditionQuery() throws IOException { + String errorMessage = "Result does not exist to the condition " + + "NOT (gender='m' AND NOT (age > 25 OR account_number > 5)) " + + "OR (NOT gender='f' AND NOT (age > 30 OR account_number < 8))"; + + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * " + + "FROM %s " + + "WHERE NOT (gender='m' AND NOT (age > 25 OR account_number > 5)) " + + "OR (NOT gender='f' AND NOT (age > 30 OR account_number < 8))", + TEST_INDEX_ACCOUNT)); + + JSONArray hits = getHits(response); + Assert.assertNotEquals(0, hits.length()); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + JSONObject source = getSource(hit); + + String gender = source.getString("gender").toLowerCase(); + int age = source.getInt("age"); + int accountNumber = source.getInt("account_number"); + + Assert.assertTrue(errorMessage, + !(gender.equals("m") && !(age > 25 || accountNumber > 5)) + || (!gender.equals("f") && !(age > 30 || accountNumber < 8))); + } + } + + @Test + @SuppressWarnings("unchecked") + public void orderByAscTest() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT age FROM %s ORDER BY age ASC LIMIT 1000", + TEST_INDEX_ACCOUNT)); + + JSONArray hits = getHits(response); + ArrayList ages = new ArrayList<>(); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + JSONObject source = getSource(hit); + + ages.add(source.getInt("age")); + } + + ArrayList sortedAges = (ArrayList) ages.clone(); + Collections.sort(sortedAges); + Assert.assertEquals("The list is not in ascending order", sortedAges, ages); + } + + @Test + public void orderByDescTest() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT age FROM %s ORDER BY age DESC LIMIT 1000", + TEST_INDEX_ACCOUNT)); + assertResponseForOrderByTest(response); + } + + @Test + public void orderByDescUsingTableAliasTest() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT a.age FROM %s a ORDER BY a.age DESC LIMIT 1000", + TEST_INDEX_ACCOUNT)); + assertResponseForOrderByTest(response); + } + + @SuppressWarnings("unchecked") + private void assertResponseForOrderByTest(JSONObject response) { + JSONArray hits = getHits(response); + ArrayList ages = new ArrayList<>(); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + JSONObject source = getSource(hit); + + ages.add(source.getInt("age")); + } + + ArrayList sortedAges = (ArrayList) ages.clone(); + Collections.sort(sortedAges, Collections.reverseOrder()); + Assert.assertEquals("The list is not in ascending order", sortedAges, ages); + } + + @Test + @SuppressWarnings("unchecked") + public void orderByAscFieldWithSpaceTest() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * " + + "FROM %s " + + "WHERE `test field` IS NOT null " + + "ORDER BY `test field` ASC " + + "LIMIT 1000", + TestsConstants.TEST_INDEX_PHRASE)); + + JSONArray hits = getHits(response); + ArrayList testFields = new ArrayList<>(); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + JSONObject source = getSource(hit); + + testFields.add(source.getInt("test field")); + } + + ArrayList sortedTestFields = (ArrayList) testFields.clone(); + Collections.sort(sortedTestFields); + Assert.assertEquals("The list is not in ascending order", sortedTestFields, testFields); + } + + @Test + public void testWhereWithBoolEqualsTrue() throws IOException { + JSONObject response = executeQuery( + StringUtils.format( + "SELECT * " + + "FROM %s " + + "WHERE male = true " + + "LIMIT 5", + TestsConstants.TEST_INDEX_BANK) + ); + + checkResponseSize(response, BANK_INDEX_MALE_TRUE); + } + + @Test + public void testWhereWithBoolEqualsTrueAndGroupBy() throws IOException { + JSONObject response = executeQuery( + StringUtils.format( + "SELECT * " + + "FROM %s " + + "WHERE male = true " + + "GROUP BY balance " + + "LIMIT 5", + TestsConstants.TEST_INDEX_BANK) + ); + + checkAggregationResponseSize(response, BANK_INDEX_MALE_TRUE); + } + + @Test + public void testWhereWithBoolEqualsTrueAndOrderBy() throws IOException { + JSONObject response = executeQuery( + StringUtils.format( + "SELECT * " + + "FROM %s " + + "WHERE male = true " + + "ORDER BY age " + + "LIMIT 5", + TestsConstants.TEST_INDEX_BANK) + ); + + checkResponseSize(response, BANK_INDEX_MALE_TRUE); + } + + @Test + public void testWhereWithBoolIsTrue() throws IOException { + JSONObject response = executeQuery( + StringUtils.format( + "SELECT * " + + "FROM %s " + + "WHERE male IS true " + + "GROUP BY balance " + + "LIMIT 5", + TestsConstants.TEST_INDEX_BANK) + ); + + checkAggregationResponseSize(response, BANK_INDEX_MALE_TRUE); + } + + @Test + public void testWhereWithBoolIsNotTrue() throws IOException { + JSONObject response = executeQuery( + StringUtils.format( + "SELECT * " + + "FROM %s " + + "WHERE male IS NOT true " + + "GROUP BY balance " + + "LIMIT 5", + TestsConstants.TEST_INDEX_BANK) + ); + + checkAggregationResponseSize(response, BANK_INDEX_MALE_FALSE); + } + + @Test + public void testWhereWithBoolEqualsFalse() throws IOException { + JSONObject response = executeQuery( + StringUtils.format( + "SELECT * " + + "FROM %s " + + "WHERE male = false " + + "LIMIT 5", + TestsConstants.TEST_INDEX_BANK) + ); + + checkResponseSize(response, BANK_INDEX_MALE_FALSE); + } + + @Test + public void testWhereWithBoolEqualsFalseAndGroupBy() throws IOException { + JSONObject response = executeQuery( + StringUtils.format( + "SELECT * " + + "FROM %s " + + "WHERE male = false " + + "GROUP BY balance " + + "LIMIT 5", + TestsConstants.TEST_INDEX_BANK) + ); + + checkAggregationResponseSize(response, BANK_INDEX_MALE_FALSE); + } + + @Test + public void testWhereWithBoolEqualsFalseAndOrderBy() throws IOException { + JSONObject response = executeQuery( + StringUtils.format( + "SELECT * " + + "FROM %s " + + "WHERE male = false " + + "ORDER BY age " + + "LIMIT 5", + TestsConstants.TEST_INDEX_BANK) + ); + + checkResponseSize(response, BANK_INDEX_MALE_FALSE); + } + + @Test + public void testWhereWithBoolIsFalse() throws IOException { + JSONObject response = executeQuery( + StringUtils.format( + "SELECT * " + + "FROM %s " + + "WHERE male IS false " + + "GROUP BY balance " + + "LIMIT 5", + TestsConstants.TEST_INDEX_BANK) + ); + + checkAggregationResponseSize(response, BANK_INDEX_MALE_FALSE); + } + + @Test + public void testWhereWithBoolIsNotFalse() throws IOException { + JSONObject response = executeQuery( + StringUtils.format( + "SELECT * " + + "FROM %s " + + "WHERE male IS NOT false " + + "GROUP BY balance " + + "LIMIT 5", + TestsConstants.TEST_INDEX_BANK) + ); + + checkAggregationResponseSize(response, BANK_INDEX_MALE_TRUE); + } + + @Test + public void testMultiPartWhere() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * " + + "FROM %s " + + "WHERE (firstname LIKE 'opal' OR firstname LIKE 'rodriquez') " + + "AND (state like 'oh' OR state like 'hi')", + TEST_INDEX_ACCOUNT)); + + Assert.assertEquals(2, getTotalHits(response)); + } + + @Test + public void testMultiPartWhere2() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * " + + "FROM %s " + + "WHERE ((account_number > 200 AND account_number < 300) OR gender LIKE 'm') " + + "AND (state LIKE 'hi' OR address LIKE 'avenue')", + TEST_INDEX_ACCOUNT)); + + Assert.assertEquals(127, getTotalHits(response)); + } + + @Test + public void testMultiPartWhere3() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * " + + "FROM %s " + + "WHERE ((account_number > 25 AND account_number < 75) AND age >35 ) " + + "AND (state LIKE 'md' OR (address LIKE 'avenue' OR address LIKE 'street'))", + TEST_INDEX_ACCOUNT)); + + Assert.assertEquals(7, getTotalHits(response)); + } + + @Test + public void filterPolygonTest() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * " + + "FROM %s " + + "WHERE GEO_INTERSECTS(place,'POLYGON ((102 2, 103 2, 103 3, 102 3, 102 2))')", + TestsConstants.TEST_INDEX_LOCATION)); + + JSONArray hits = getHits(response); + Assert.assertEquals(1, getTotalHits(response)); + + JSONObject hit = hits.getJSONObject(0); + Assert.assertEquals("bigSquare", getSource(hit).getString("description")); + } + + @Test + public void boundingBox() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, + "SELECT * FROM %s WHERE GEO_BOUNDING_BOX(center, 100.0, 1.0, 101, 0.0)", + TestsConstants.TEST_INDEX_LOCATION)); + + JSONArray hits = getHits(response); + Assert.assertEquals(1, getTotalHits(response)); + + JSONObject hit = hits.getJSONObject(0); + Assert.assertEquals("square", getSource(hit).getString("description")); + } + + @Test + public void geoDistance() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, + "SELECT * FROM %s WHERE GEO_DISTANCE(center, '1km', 100.5, 0.500001)", + TestsConstants.TEST_INDEX_LOCATION)); + + JSONArray hits = getHits(response); + Assert.assertEquals(1, getTotalHits(response)); + + JSONObject hit = hits.getJSONObject(0); + Assert.assertEquals("square", getSource(hit).getString("description")); + } + + @Test + public void geoPolygon() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, + "SELECT * FROM %s WHERE GEO_POLYGON(center, 100,0, 100.5, 2, 101.0,0)", + TestsConstants.TEST_INDEX_LOCATION)); + + JSONArray hits = getHits(response); + Assert.assertEquals(1, getTotalHits(response)); + + JSONObject hit = hits.getJSONObject(0); + Assert.assertEquals("square", getSource(hit).getString("description")); + } + + @Ignore + @Test + public void escapedCharactersCheck() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * " + + "FROM %s " + + "WHERE MATCH_PHRASE(nickname, 'Daenerys \"Stormborn\"') " + + "LIMIT 1000", + TestsConstants.TEST_INDEX_GAME_OF_THRONES)); + + Assert.assertEquals(1, getTotalHits(response)); + } + + @Test + public void complexObjectSearch() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * " + + "FROM %s " + + "WHERE MATCH_PHRASE(name.firstname, 'Jaime') " + + "LIMIT 1000", + TestsConstants.TEST_INDEX_GAME_OF_THRONES)); + + Assert.assertEquals(1, getTotalHits(response)); + } + + @Test + public void complexObjectReturnField() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT parents.father " + + "FROM %s " + + "WHERE MATCH_PHRASE(name.firstname, 'Brandon') " + + "LIMIT 1000", + TestsConstants.TEST_INDEX_GAME_OF_THRONES)); + + JSONArray hits = getHits(response); + Assert.assertEquals(1, getTotalHits(response)); + + JSONObject hit = hits.getJSONObject(0); + Assert.assertEquals("Eddard", getSource(hit).getJSONObject("parents").getString("father")); + } + + /** + * TODO: Fields prefixed with @ gets converted to SQLVariantRefExpr instead of SQLIdentifierExpr + * Either change SQLVariantRefExpr to SQLIdentifierExpr + * Or handle the special case for SQLVariantRefExpr + */ + @Ignore + @Test + public void queryWithAtFieldOnWhere() throws IOException { + JSONObject response = executeQuery(String.format(Locale.ROOT, + "SELECT * FROM %s where @wolf = 'Summer' LIMIT 1000", TEST_INDEX_GAME_OF_THRONES)); + Assert.assertEquals(1, getTotalHits(response)); + JSONObject hit = getHits(response).getJSONObject(0); + Assert.assertEquals("Summer", hit.get("@wolf")); + Assert.assertEquals("Brandon", hit.query("name/firstname")); + } + + @Test + public void queryWithDotAtStartOfIndexName() throws Exception { + TestUtils.createHiddenIndexByRestClient(client(), ".bank", null); + TestUtils.loadDataByRestClient(client(), ".bank", "/src/test/resources/.bank.json"); + + String response = executeQuery("SELECT education FROM .bank WHERE account_number = 12345", + "jdbc"); + Assert.assertTrue(response.contains("PhD")); + } + + @Test + public void notLikeTests() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT name " + + "FROM %s " + + "WHERE name.firstname NOT LIKE 'd%%' AND name IS NOT NULL " + + "LIMIT 1000", + TestsConstants.TEST_INDEX_GAME_OF_THRONES)); + + JSONArray hits = getHits(response); + Assert.assertEquals(3, getTotalHits(response)); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + JSONObject source = getSource(hit); + + String name = source.getJSONObject("name").getString("firstname"); + Assert + .assertFalse(String.format(Locale.ROOT, "Name [%s] should not match pattern [d%%]", name), + name.startsWith("d")); + } + } + + @Test + public void isNullTest() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT name " + + "FROM %s " + + "WHERE nickname IS NULL " + + "LIMIT 1000", + TestsConstants.TEST_INDEX_GAME_OF_THRONES)); + + Assert.assertEquals(6, getTotalHits(response)); + } + + @Test + public void isNotNullTest() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT name " + + "FROM %s " + + "WHERE nickname IS NOT NULL " + + "LIMIT 1000", + TestsConstants.TEST_INDEX_GAME_OF_THRONES)); + + Assert.assertEquals(1, getTotalHits(response)); + } + + @Test + public void innerQueryTest() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * " + + "FROM %s D " + + "WHERE holdersName IN (SELECT firstname " + + "FROM %s " + + "WHERE firstname = 'Hattie')", + TestsConstants.TEST_INDEX_DOG, TEST_INDEX_ACCOUNT)); + + JSONArray hits = getHits(response); + Assert.assertEquals(1, hits.length()); + + JSONObject hit = hits.getJSONObject(0); + JSONObject source = getSource(hit); + Assert.assertEquals("snoopy", source.getString("D.dog_name")); + Assert.assertEquals("Hattie", source.getString("D.holdersName")); + Assert.assertEquals(4, source.getInt("D.age")); + } + + @Ignore + @Test + public void twoSubQueriesTest() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * " + + "FROM %s " + + "WHERE holdersName IN (SELECT firstname " + + "FROM %s " + + "WHERE firstname = 'Hattie') " + + "AND age IN (SELECT name.ofHisName " + + "FROM %s " + + "WHERE name.firstname <> 'Daenerys' " + + "AND name.ofHisName IS NOT NULL) ", + TestsConstants.TEST_INDEX_DOG, + TEST_INDEX_ACCOUNT, + TestsConstants.TEST_INDEX_GAME_OF_THRONES)); + + JSONArray hits = getHits(response); + Assert.assertEquals(1, hits.length()); + + JSONObject hit = hits.getJSONObject(0); + JSONObject source = getSource(hit); + Assert.assertEquals("snoopy", source.getString("dog_name")); + Assert.assertEquals("Hattie", source.getString("holdersName")); + Assert.assertEquals(4, source.getInt("age")); + } + + @Ignore + @Test + public void inTermsSubQueryTest() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * " + + "FROM %s " + + "WHERE age = IN_TERMS (SELECT name.ofHisName " + + "FROM %s " + + "WHERE name.firstname <> 'Daenerys' " + + "AND name.ofHisName IS NOT NULL)", + TestsConstants.TEST_INDEX_DOG, TestsConstants.TEST_INDEX_GAME_OF_THRONES)); + + JSONArray hits = getHits(response); + Assert.assertEquals(1, hits.length()); + + JSONObject hit = hits.getJSONObject(0); + JSONObject source = getSource(hit); + Assert.assertEquals("snoopy", source.getString("dog_name")); + Assert.assertEquals("Hattie", source.getString("holdersName")); + Assert.assertEquals(4, source.getInt("age")); + } + + @Ignore + @Test + public void idsQueryOneId() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * FROM %s WHERE _id = IDS_QUERY(dog, 1)", + TestsConstants.TEST_INDEX_DOG)); + + JSONArray hits = getHits(response); + Assert.assertEquals(1, hits.length()); + + JSONObject hit = hits.getJSONObject(0); + JSONObject source = getSource(hit); + Assert.assertEquals("rex", source.getString("dog_name")); + Assert.assertEquals("Daenerys", source.getString("holdersName")); + Assert.assertEquals(2, hit.getInt("age")); + } + + @Ignore + @Test + public void idsQueryMultipleId() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * FROM %s WHERE _id = IDS_QUERY(dog, 1, 2, 3)", + TestsConstants.TEST_INDEX_DOG)); + + JSONArray hits = getHits(response); + Assert.assertEquals(1, hits.length()); + + JSONObject hit = hits.getJSONObject(0); + JSONObject source = getSource(hit); + Assert.assertEquals("rex", source.getString("dog_name")); + Assert.assertEquals("Daenerys", source.getString("holdersName")); + Assert.assertEquals(2, hit.getInt("age")); + } + + @Ignore + @Test + public void idsQuerySubQueryIds() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * " + + "FROM %s " + + "WHERE _id = IDS_QUERY(dog, (SELECT name.ofHisName " + + "FROM %s " + + "WHERE name.firstname <> 'Daenerys' " + + "AND name.ofHisName IS NOT NULL))", + TestsConstants.TEST_INDEX_DOG, TestsConstants.TEST_INDEX_GAME_OF_THRONES)); + + JSONArray hits = getHits(response); + Assert.assertEquals(1, hits.length()); + + JSONObject hit = hits.getJSONObject(0); + JSONObject source = getSource(hit); + Assert.assertEquals("rex", source.getString("dog_name")); + Assert.assertEquals("Daenerys", source.getString("holdersName")); + Assert.assertEquals(2, hit.getInt("age")); + } + + @Test + public void nestedEqualsTestFieldNormalField() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * FROM %s WHERE nested(message.info)='b'", + TestsConstants.TEST_INDEX_NESTED_TYPE)); + + Assert.assertEquals(1, getTotalHits(response)); + } + + @Test + public void nestedEqualsTestFieldInsideArrays() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * FROM %s WHERE nested(message.info) = 'a'", + TestsConstants.TEST_INDEX_NESTED_TYPE)); + + Assert.assertEquals(2, getTotalHits(response)); + } + + @Ignore // Seems like we don't support nested with IN, throwing IllegalArgumentException + @Test + public void nestedOnInQuery() throws IOException { + JSONObject response = executeQuery(String.format(Locale.ROOT, + "SELECT * FROM %s where nested(message.info) IN ('a','b')", TEST_INDEX_NESTED_TYPE)); + Assert.assertEquals(3, getTotalHits(response)); + } + + @Test + public void complexNestedQueryBothOnSameObject() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * " + + "FROM %s " + + "WHERE nested('message', message.info = 'a' AND message.author ='i')", + TestsConstants.TEST_INDEX_NESTED_TYPE)); + + Assert.assertEquals(1, getTotalHits(response)); + } + + @Test + public void complexNestedQueryNotBothOnSameObject() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * " + + "FROM %s " + + "WHERE nested('message', message.info = 'a' AND message.author ='h')", + TestsConstants.TEST_INDEX_NESTED_TYPE)); + + Assert.assertEquals(0, getTotalHits(response)); + } + + @Test + public void nestedOnInTermsQuery() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT * " + + "FROM %s " + + "WHERE nested(message.info) = IN_TERMS('a', 'b')", + TestsConstants.TEST_INDEX_NESTED_TYPE)); + + Assert.assertEquals(3, getTotalHits(response)); + } + + // TODO Uncomment these after problem with loading join index is resolved +// @Test +// public void childrenEqualsTestFieldNormalField() throws IOException { +// JSONObject response = executeQuery( +// String.format(Locale.ROOT, "SELECT * " + +// "FROM %s/joinType " + +// "WHERE children(childrenType, info) = 'b'", TestsConstants.TEST_INDEX_JOIN_TYPE)); +// +// Assert.assertEquals(1, getTotalHits(response)); +// } +// +// @Test +// public void childrenOnInQuery() throws IOException { +// JSONObject response = executeQuery( +// String.format(Locale.ROOT, "SELECT * " + +// "FROM %s/joinType " + +// "WHERE children(childrenType, info) IN ('a', 'b')", +// TestsConstants.TEST_INDEX_JOIN_TYPE)); +// +// Assert.assertEquals(2, getTotalHits(response)); +// } +// +// @Test +// public void complexChildrenQueryBothOnSameObject() throws IOException { +// JSONObject response = executeQuery( +// String.format(Locale.ROOT, "SELECT * " + +// "FROM %s/joinType " + +// "WHERE children(childrenType, info = 'a' AND author ='e')", +// TestsConstants.TEST_INDEX_JOIN_TYPE)); +// +// Assert.assertEquals(1, getTotalHits(response)); +// } +// +// @Test +// public void complexChildrenQueryNotOnSameObject() throws IOException { +// JSONObject response = executeQuery( +// String.format(Locale.ROOT, "SELECT * " + +// "FROM %s/joinType " + +// "WHERE children(childrenType, info = 'a' AND author ='j')", +// TestsConstants.TEST_INDEX_JOIN_TYPE)); +// +// Assert.assertEquals(0, getTotalHits(response)); +// } +// +// @Test +// public void childrenOnInTermsQuery() throws IOException { +// JSONObject response = executeQuery( +// String.format(Locale.ROOT, "SELECT * " + +// "FROM %s/joinType " + +// "WHERE children(childrenType, info) = IN_TERMS(a, b)", +// TestsConstants.TEST_INDEX_JOIN_TYPE)); +// +// Assert.assertEquals(2, getTotalHits(response)); +// } + + @Ignore // the hint does not really work, NoSuchIndexException is thrown + @Test + public void multipleIndicesOneNotExistWithHint() throws IOException { + + JSONObject response = executeQuery(String + .format(Locale.ROOT, "SELECT /*! IGNORE_UNAVAILABLE */ * FROM %s,%s ", TEST_INDEX_ACCOUNT, + "badindex")); + + Assert.assertTrue(getTotalHits(response) > 0); + } + + @Test + public void multipleIndicesOneNotExistWithoutHint() throws IOException { + try { + executeQuery( + String.format(Locale.ROOT, "SELECT * FROM %s, %s", TEST_INDEX_ACCOUNT, "badindex")); + Assert.fail("Expected exception, but call succeeded"); + } catch (ResponseException e) { + Assert.assertEquals(RestStatus.BAD_REQUEST.getStatus(), + e.getResponse().getStatusLine().getStatusCode()); + final String entity = TestUtils.getResponseBody(e.getResponse()); + Assert.assertThat(entity, containsString("\"type\": \"IndexNotFoundException\"")); + } + } + + // TODO Find way to check routing() without SearchRequestBuilder + // to properly update these tests to ESIntegTestCase format +// @Test +// public void routingRequestOneRounting() throws IOException { +// SqlElasticSearchRequestBuilder request = getRequestBuilder(String.format(Locale.ROOT, +// "SELECT /*! ROUTINGS(hey) */ * FROM %s ", TEST_INDEX_ACCOUNT)); +// SearchRequestBuilder searchRequestBuilder = (SearchRequestBuilder) request.getBuilder(); +// Assert.assertEquals("hey",searchRequestBuilder.request().routing()); +// } +// +// @Test +// public void routingRequestMultipleRountings() throws IOException { +// SqlElasticSearchRequestBuilder request = getRequestBuilder(String.format(Locale.ROOT, +// "SELECT /*! ROUTINGS(hey,bye) */ * FROM %s ", TEST_INDEX_ACCOUNT)); +// SearchRequestBuilder searchRequestBuilder = (SearchRequestBuilder) request.getBuilder(); +// Assert.assertEquals("hey,bye",searchRequestBuilder.request().routing()); +// } + + @Ignore // Getting parser error: syntax error, expect RPAREN, actual IDENTIFIER insert_time + @Test + public void scriptFilterNoParams() throws IOException { + + JSONObject result = executeQuery(String.format(Locale.ROOT, + "SELECT insert_time FROM %s where script('doc[\\'insert_time\''].date.hourOfDay==16') " + + "and insert_time <'2014-08-21T00:00:00.000Z'", TEST_INDEX_ONLINE)); + Assert.assertEquals(237, getTotalHits(result)); + } + + @Ignore // Getting parser error: syntax error, expect RPAREN, actual IDENTIFIER insert_time + @Test + public void scriptFilterWithParams() throws IOException { + + JSONObject result = executeQuery(String.format(Locale.ROOT, + "SELECT insert_time FROM %s where script('doc[\\'insert_time\''].date.hourOfDay==x','x'=16) " + + "and insert_time <'2014-08-21T00:00:00.000Z'", TEST_INDEX_ONLINE)); + Assert.assertEquals(237, getTotalHits(result)); + } + + @Test + public void highlightPreTagsAndPostTags() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, + "SELECT /*! HIGHLIGHT(phrase, pre_tags : [''], post_tags : ['']) */ " + + "* FROM %s " + + "WHERE phrase LIKE 'fox' " + + "ORDER BY _score", TestsConstants.TEST_INDEX_PHRASE)); + + JSONArray hits = getHits(response); + for (int i = 0; i < hits.length(); i++) { + JSONObject hit = hits.getJSONObject(i); + JSONObject highlightFields = hit.getJSONObject("highlight"); + + String phrase = highlightFields.getJSONArray("phrase").getString(0); + Assert.assertTrue(phrase.contains("fox")); + } + } + + @Ignore + @Test + public void fieldCollapsingTest() throws IOException { + JSONObject response = executeQuery( + String.format(Locale.ROOT, "SELECT /*! COLLAPSE({\"field\":\"age\"," + + "\"inner_hits\":{\"name\": \"account\"," + + "\"size\":1," + + "\"sort\":[{\"age\":\"asc\"}]}," + + "\"max_concurrent_group_searches\": 4}) */ " + + "* FROM %s", TEST_INDEX_ACCOUNT)); + + JSONArray hits = getHits(response); + Assert.assertEquals(21, hits.length()); + } + + @Test + public void backticksQuotedIndexNameTest() throws Exception { + TestUtils.createIndexByRestClient(client(), "bank_unquote", null); + TestUtils + .loadDataByRestClient(client(), "bank", "/src/test/resources/bank_for_unquote_test.json"); + + JSONArray hits = getHits(executeQuery("SELECT lastname FROM `bank`")); + Object responseIndex = ((JSONObject) hits.get(0)).query("/_index"); + assertEquals("bank", responseIndex); + + assertEquals( + executeQuery("SELECT lastname FROM bank", "jdbc"), + executeQuery("SELECT `bank`.`lastname` FROM `bank`", "jdbc") + ); + + assertEquals( + executeQuery( + "SELECT `b`.`age` AS `AGE`, AVG(`b`.`balance`) FROM `bank` AS `b` " + + "WHERE ABS(`b`.`age`) > 20 GROUP BY `b`.`age` ORDER BY `b`.`age`", + "jdbc"), + executeQuery("SELECT b.age AS AGE, AVG(balance) FROM bank AS b " + + "WHERE ABS(age) > 20 GROUP BY b.age ORDER BY b.age", + "jdbc") + ); + } + + @Test + public void backticksQuotedFieldNamesTest() { + String expected = executeQuery(StringUtils.format("SELECT b.lastname FROM %s " + + "AS b ORDER BY age LIMIT 3", TestsConstants.TEST_INDEX_BANK), "jdbc"); + String quotedFieldResult = executeQuery(StringUtils.format("SELECT b.`lastname` FROM %s " + + "AS b ORDER BY age LIMIT 3", TestsConstants.TEST_INDEX_BANK), "jdbc"); + + assertEquals(expected, quotedFieldResult); + } + + @Test + public void backticksQuotedAliasTest() { + String expected = executeQuery(StringUtils.format("SELECT b.lastname FROM %s " + + "AS b ORDER BY age LIMIT 3", TestsConstants.TEST_INDEX_BANK), "jdbc"); + String quotedAliasResult = executeQuery(StringUtils.format("SELECT `b`.lastname FROM %s" + + " AS `b` ORDER BY age LIMIT 3", TestsConstants.TEST_INDEX_BANK), "jdbc"); + String quotedAliasAndFieldResult = + executeQuery(StringUtils.format("SELECT `b`.`lastname` FROM %s " + + "AS `b` ORDER BY age LIMIT 3", TestsConstants.TEST_INDEX_BANK), "jdbc"); + + assertEquals(expected, quotedAliasResult); + assertEquals(expected, quotedAliasAndFieldResult); + } + + @Test + public void backticksQuotedAliasWithSpecialCharactersTest() { + String expected = executeQuery(StringUtils.format("SELECT b.lastname FROM %s " + + "AS b ORDER BY age LIMIT 3", TestsConstants.TEST_INDEX_BANK), "jdbc"); + String specialCharAliasResult = + executeQuery(StringUtils.format("SELECT `b k`.lastname FROM %s " + + "AS `b k` ORDER BY age LIMIT 3", TestsConstants.TEST_INDEX_BANK), "jdbc"); + + assertEquals(expected, specialCharAliasResult); + } + + @Test + public void backticksQuotedAliasInJDBCResponseTest() { + String query = StringUtils.format("SELECT `b`.`lastname` AS `name` FROM %s AS `b` " + + "ORDER BY age LIMIT 3", TestsConstants.TEST_INDEX_BANK); + String response = executeQuery(query, "jdbc"); + + assertTrue(response.contains("\"alias\": \"name\"")); + } + + @Test + public void caseWhenSwitchTest() throws IOException { + JSONObject response = executeQuery("SELECT CASE age " + + "WHEN '30' THEN '1' " + + "WHEN '40' THEN '2' " + + "ELSE '0' END AS cases FROM " + TEST_INDEX_ACCOUNT + " WHERE age IS NOT NULL"); + JSONObject hit = getHits(response).getJSONObject(0); + String age = hit.query("/_source/age").toString(); + String cases = age.equals("30") ? "1" : age.equals("40") ? "2" : "0"; + + assertThat(cases, equalTo(hit.query("/fields/cases/0"))); + } + + @Test + public void caseWhenJdbcResponseTest() { + String response = executeQuery("SELECT CASE age " + + "WHEN '30' THEN 'age is 30' " + + "WHEN '40' THEN 'age is 40' " + + "ELSE 'NA' END AS cases FROM " + TEST_INDEX_ACCOUNT + " WHERE age is not null", "jdbc"); + assertTrue( + response.contains("age is 30") || + response.contains("age is 40") || + response.contains("NA") + ); + } + + @Test + public void functionInCaseFieldShouldThrowESExceptionDueToIllegalScriptInJdbc() { + String response = executeQuery( + "select case lower(firstname) when 'amber' then '1' else '2' end as cases from " + + TEST_INDEX_ACCOUNT, + "jdbc"); + queryInJdbcResponseShouldIndicateESException(response, "SearchPhaseExecutionException", + "For more details, please send request for Json format"); + } + + @Test + public void functionCallWithIllegalScriptShouldThrowESExceptionInJdbc() { + String response = executeQuery("select log(balance + 2) from " + TEST_INDEX_BANK, + "jdbc"); + queryInJdbcResponseShouldIndicateESException(response, "SearchPhaseExecutionException", + "please send request for Json format to see the raw response from elasticsearch engine."); + } + + @Ignore("Goes in different route, does not call PrettyFormatRestExecutor.execute methods." + + "The performRequest method in RestClient doesn't throw any exceptions for null value fields in script") + @Test + public void functionArgWithNullValueFieldShouldThrowESExceptionInJdbc() { + String response = executeQuery( + "select log(balance) from " + TEST_INDEX_BANK_WITH_NULL_VALUES, "jdbc"); + queryInJdbcResponseShouldIndicateESException(response, "SearchPhaseExecutionException", + "For more details, please send request for Json format"); + } + + private void queryInJdbcResponseShouldIndicateESException(String response, String exceptionType, + String... errMsgs) { + Assert.assertThat(response, containsString(exceptionType)); + for (String errMsg : errMsgs) { + Assert.assertThat(response, containsString(errMsg)); + } + } + + private String getScrollId(JSONObject response) { + return response.getString("_scroll_id"); + } + + private void checkResponseSize(JSONObject response, int sizeCheck) { + JSONArray queryResponse = getHits(response); + Assert.assertThat(queryResponse.length(), equalTo(sizeCheck)); + } + + private void checkAggregationResponseSize(JSONObject response, int sizeCheck) { + JSONArray queryResponse = (JSONArray) response.query("/aggregations/balance/buckets"); + Assert.assertThat(queryResponse.length(), equalTo(sizeCheck)); + } + + private void checkSelectAllAndFieldResponseSize(JSONObject response) { + String[] arr = + new String[] {"account_number", "firstname", "address", "birthdate", "gender", "city", + "lastname", + "balance", "employer", "state", "age", "email", "male"}; + Set expectedSource = new HashSet<>(Arrays.asList(arr)); + + JSONArray hits = getHits(response); + Assert.assertTrue(hits.length() > 0); + for (int i = 0; i < hits.length(); ++i) { + JSONObject hit = hits.getJSONObject(i); + Assert.assertEquals(expectedSource, getSource(hit).keySet()); + } + } + + private void checkSelectAllAndFieldAggregationResponseSize(JSONObject response, String field) { + JSONObject fieldAgg = (response.getJSONObject("aggregations")).getJSONObject(field); + JSONArray buckets = fieldAgg.getJSONArray("buckets"); + Assert.assertTrue(buckets.length() == 6); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/RestIntegTestCase.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/RestIntegTestCase.java new file mode 100644 index 0000000000..961485bd85 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/RestIntegTestCase.java @@ -0,0 +1,326 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.createIndexByRestClient; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getAccountIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getBankIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getBankWithNullValuesIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getDateIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getDogIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getDogs2IndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getDogs3IndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getEmployeeNestedTypeIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getGameOfThronesIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getJoinTypeIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getLocationIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getNestedTypeIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getOdbcIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getOrderIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getPeople2IndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getPhraseIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getWeblogsIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.isIndexExist; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.loadDataByRestClient; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.PERSISTENT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TRANSIENT; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import javax.management.MBeanServerInvocationHandler; +import javax.management.ObjectName; +import javax.management.remote.JMXConnector; +import javax.management.remote.JMXConnectorFactory; +import javax.management.remote.JMXServiceURL; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; + +/** + * SQL plugin integration test base class (migrated from SQLIntegTestCase) + *

+ * The execution of order is as follows: + *

+ * ESRestTestCase: 1) initClient() N+1) closeClient() + * \ / + * SQLIntegTestCase: 2) setUpIndices() -> 4) setUpIndices() ... -> N) cleanUpIndices() + * \ \ + * XXXTIT: 3) init() 5) init() + *

+ * TODO: this base class should extends ODFERestTestCase + */ +public abstract class RestIntegTestCase extends ESRestTestCase { + + @Before + public void setUpIndices() throws Exception { + if (client() == null) { + initClient(); + } + + increaseScriptMaxCompilationsRate(); + init(); + } + + @Override + protected boolean preserveClusterUponCompletion() { + return true; + } + + /** + * We need to be able to dump the jacoco coverage before cluster is shut down. + * The new internal testing framework removed some of the gradle tasks we were listening to + * to choose a good time to do it. This will dump the executionData to file after each test. + * TODO: This is also currently just overwriting integTest.exec with the updated execData without + * resetting after writing each time. This can be improved to either write an exec file per test + * or by letting jacoco append to the file + */ + public interface IProxy { + byte[] getExecutionData(boolean reset); + + void dump(boolean reset); + + void reset(); + } + + @AfterClass + public static void dumpCoverage() { + // jacoco.dir is set in sqlplugin-coverage.gradle, if it doesn't exist we don't + // want to collect coverage so we can return early + String jacocoBuildPath = System.getProperty("jacoco.dir"); + if (Strings.isNullOrEmpty(jacocoBuildPath)) { + return; + } + + String serverUrl = "service:jmx:rmi:///jndi/rmi://127.0.0.1:7777/jmxrmi"; + try (JMXConnector connector = JMXConnectorFactory.connect(new JMXServiceURL(serverUrl))) { + IProxy proxy = MBeanServerInvocationHandler.newProxyInstance( + connector.getMBeanServerConnection(), new ObjectName("org.jacoco:type=Runtime"), + IProxy.class, + false); + + Path path = Paths.get(jacocoBuildPath + "/integTest.exec"); + Files.write(path, proxy.getExecutionData(false)); + } catch (Exception ex) { + throw new RuntimeException("Failed to dump coverage", ex); + } + } + + /** + * As JUnit JavaDoc says: + * "The @AfterClass methods declared in superclasses will be run after those of the current class." + * So this method is supposed to run before closeClients() in parent class. + */ + @AfterClass + public static void cleanUpIndices() throws IOException { + wipeAllIndices(); + wipeAllClusterSettings(); + } + + /** + * Make it thread-safe in case tests are running in parallel but does not guarantee + * if test like DeleteIT that mutates cluster running in parallel. + */ + protected synchronized void loadIndex(Index index) throws IOException { + String indexName = index.getName(); + String mapping = index.getMapping(); + String dataSet = index.getDataSet(); + + if (!isIndexExist(client(), indexName)) { + createIndexByRestClient(client(), indexName, mapping); + loadDataByRestClient(client(), indexName, dataSet); + } + } + + /** + * Increase script.max_compilations_rate to large enough, which is only 75/5min by default. + * This issue is due to our painless script not using params passed to compiled script. + */ + private void increaseScriptMaxCompilationsRate() throws IOException { + updateClusterSetting("script.max_compilations_rate", "10000/1m", false); + } + + /** + * Provide for each test to load test index, data and other setup work + */ + protected void init() throws Exception { + } + + protected static void updateClusterSetting(String settingKey, Object value) throws IOException { + updateClusterSetting(settingKey, value, true); + } + + protected static void updateClusterSetting(String settingKey, Object value, boolean persistent) + throws IOException { + String property = persistent ? PERSISTENT : TRANSIENT; + XContentBuilder builder = XContentFactory + .jsonBuilder() + .startObject() + .startObject(property) + .field(settingKey, value) + .endObject() + .endObject(); + Request request = new Request("PUT", "_cluster/settings"); + request.setJsonEntity(Strings.toString(builder)); + Response response = client().performRequest(request); + Assert + .assertEquals(RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + } + + protected static void wipeAllClusterSettings() throws IOException { + updateClusterSetting("*", null, true); + updateClusterSetting("*", null, false); + } + + /** + * Enum for associating test index with relevant mapping and data. + */ + public enum Index { + ONLINE(TestsConstants.TEST_INDEX_ONLINE, + "online", + null, + "src/test/resources/online.json"), + ACCOUNT(TestsConstants.TEST_INDEX_ACCOUNT, + "account", + getAccountIndexMapping(), + "src/test/resources/accounts.json"), + PHRASE(TestsConstants.TEST_INDEX_PHRASE, + "phrase", + getPhraseIndexMapping(), + "src/test/resources/phrases.json"), + DOG(TestsConstants.TEST_INDEX_DOG, + "dog", + getDogIndexMapping(), + "src/test/resources/dogs.json"), + DOGS2(TestsConstants.TEST_INDEX_DOG2, + "dog", + getDogs2IndexMapping(), + "src/test/resources/dogs2.json"), + DOGS3(TestsConstants.TEST_INDEX_DOG3, + "dog", + getDogs3IndexMapping(), + "src/test/resources/dogs3.json"), + DOGSSUBQUERY(TestsConstants.TEST_INDEX_DOGSUBQUERY, + "dog", + getDogIndexMapping(), + "src/test/resources/dogsubquery.json"), + PEOPLE(TestsConstants.TEST_INDEX_PEOPLE, + "people", + null, + "src/test/resources/peoples.json"), + PEOPLE2(TestsConstants.TEST_INDEX_PEOPLE2, + "people", + getPeople2IndexMapping(), + "src/test/resources/people2.json"), + GAME_OF_THRONES(TestsConstants.TEST_INDEX_GAME_OF_THRONES, + "gotCharacters", + getGameOfThronesIndexMapping(), + "src/test/resources/game_of_thrones_complex.json"), + SYSTEM(TestsConstants.TEST_INDEX_SYSTEM, + "systems", + null, + "src/test/resources/systems.json"), + ODBC(TestsConstants.TEST_INDEX_ODBC, + "odbc", + getOdbcIndexMapping(), + "src/test/resources/odbc-date-formats.json"), + LOCATION(TestsConstants.TEST_INDEX_LOCATION, + "location", + getLocationIndexMapping(), + "src/test/resources/locations.json"), + LOCATION_TWO(TestsConstants.TEST_INDEX_LOCATION2, + "location2", + getLocationIndexMapping(), + "src/test/resources/locations2.json"), + NESTED(TestsConstants.TEST_INDEX_NESTED_TYPE, + "nestedType", + getNestedTypeIndexMapping(), + "src/test/resources/nested_objects.json"), + NESTED_WITH_QUOTES(TestsConstants.TEST_INDEX_NESTED_WITH_QUOTES, + "nestedType", + getNestedTypeIndexMapping(), + "src/test/resources/nested_objects_quotes_in_values.json"), + EMPLOYEE_NESTED(TestsConstants.TEST_INDEX_EMPLOYEE_NESTED, + "_doc", + getEmployeeNestedTypeIndexMapping(), + "src/test/resources/employee_nested.json"), + JOIN(TestsConstants.TEST_INDEX_JOIN_TYPE, + "joinType", + getJoinTypeIndexMapping(), + "src/test/resources/join_objects.json"), + BANK(TestsConstants.TEST_INDEX_BANK, + "account", + getBankIndexMapping(), + "src/test/resources/bank.json"), + BANK_TWO(TestsConstants.TEST_INDEX_BANK_TWO, + "account_two", + getBankIndexMapping(), + "src/test/resources/bank_two.json"), + BANK_WITH_NULL_VALUES(TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES, + "account_null", + getBankWithNullValuesIndexMapping(), + "src/test/resources/bank_with_null_values.json"), + ORDER(TestsConstants.TEST_INDEX_ORDER, + "_doc", + getOrderIndexMapping(), + "src/test/resources/order.json"), + WEBLOG(TestsConstants.TEST_INDEX_WEBLOG, + "weblog", + getWeblogsIndexMapping(), + "src/test/resources/weblogs.json"), + DATE(TestsConstants.TEST_INDEX_DATE, + "dates", + getDateIndexMapping(), + "src/test/resources/dates.json"); + + private final String name; + private final String type; + private final String mapping; + private final String dataSet; + + Index(String name, String type, String mapping, String dataSet) { + this.name = name; + this.type = type; + this.mapping = mapping; + this.dataSet = dataSet; + } + + public String getName() { + return this.name; + } + + public String getType() { + return this.type; + } + + public String getMapping() { + return this.mapping; + } + + public String getDataSet() { + return this.dataSet; + } + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLFunctionsIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLFunctionsIT.java new file mode 100644 index 0000000000..e1c44a91aa --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLFunctionsIT.java @@ -0,0 +1,857 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.hitAny; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.kvDouble; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.kvInt; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.kvString; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.rows; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.schema; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyDataRows; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifySchema; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.both; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasValue; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.isEmptyOrNullString; +import static org.hamcrest.Matchers.not; + +import java.io.IOException; +import java.util.Date; +import java.util.stream.IntStream; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.hamcrest.collection.IsMapContaining; +import org.json.JSONObject; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + + +/** + * Created by allwefantasy on 8/25/16. + */ +public class SQLFunctionsIT extends SQLIntegTestCase { + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + loadIndex(Index.ONLINE); + loadIndex(Index.DATE); + } + + @Test + public void functionFieldAliasAndGroupByAlias() throws Exception { + String query = "SELECT " + + "floor(substring(address,0,3)*20) as key," + + "sum(age) cvalue FROM " + TEST_INDEX_ACCOUNT + " where address is not null " + + "group by key order by cvalue desc limit 10 "; + final JSONObject result = executeQuery(query); + + + IntStream.rangeClosed(0, 9).forEach(i -> { + Assert.assertNotNull(result.query(String.format("/aggregations/key/buckets/%d/key", i))); + Assert.assertNotNull( + result.query(String.format("/aggregations/key/buckets/%d/cvalue/value", i))); + } + ); + } + + /** + * todo fix the issue. + * + * @see https://github.com/opendistro-for-elasticsearch/sql/issues/59 + */ + @Ignore + public void normalFieldAlias() throws Exception { + + //here is a bug,csv field with spa + String query = "SELECT " + + "address as key,age from " + + TEST_INDEX_ACCOUNT + " where address is not null " + + "limit 10 "; + + assertThat( + executeQuery(query), + hitAny(kvString("/_source/key", not(isEmptyOrNullString()))) + ); + } + + @Test + public void functionAlias() throws Exception { + //here is a bug,if only script fields are included,then all fields will return; fix later + String query = "SELECT " + + "substring(address,0,3) as key,address from " + + TEST_INDEX_ACCOUNT + " where address is not null " + + "order by address desc limit 10 "; + + assertThat( + executeQuery(query), + hitAny(both(kvString("/_source/address", equalTo("863 Wythe Place"))) + .and(kvString("/fields/key/0", + equalTo("863")))) + ); + } + + @Test + public void caseChangeTest() throws IOException { + String query = "SELECT LOWER(firstname) " + + "FROM elasticsearch-sql_test_index_account " + + "WHERE UPPER(lastname)='DUKE' " + + "ORDER BY upper(lastname) "; + + assertThat( + executeQuery(query), + hitAny( + kvString("/_source/address", equalTo("880 Holmes Lane")), + kvString("/fields/LOWER(firstname)/0", equalTo("amber"))) + ); + } + + @Test + public void caseChangeTestWithLocale() throws IOException { + // Uses Turkish locale to check if we pass correct locale for case changing functions + // "IL".toLowerCase() in a Turkish locale returns "ıl" + // https://stackoverflow.com/questions/11063102/using-locales-with-javas-tolowercase-and-touppercase + + String query = "SELECT LOWER(state.keyword, 'tr') " + + "FROM elasticsearch-sql_test_index_account " + + "WHERE account_number=1"; + + assertThat( + executeQuery(query), + hitAny( + kvString("/fields/LOWER(state.keyword, 'tr')/0", equalTo("ıl"))) + ); + } + + @Test + public void caseChangeWithAggregationTest() throws IOException { + String query = "SELECT UPPER(e.firstname) AS upper, COUNT(*)" + + "FROM elasticsearch-sql_test_index_account e " + + "WHERE LOWER(e.lastname)='duke' " + + "GROUP BY upper"; + + assertThat( + executeQuery(query), + hitAny("/aggregations/upper/buckets", kvString("/key", equalTo("AMBER")))); + } + + @Test + public void castIntFieldToDoubleWithoutAliasTest() throws IOException { + String query = "SELECT CAST(age AS DOUBLE) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + + " ORDER BY age DESC LIMIT 5"; + + SearchHit[] hits = query(query).getHits(); + checkSuccessfulFieldCast(hits, "cast_age", "DOUBLE"); + for (int i = 0; i < hits.length; ++i) { + Assert.assertThat(hits[i].getFields().get("cast_age").getValue(), is(40.0)); + } + } + + @Test + public void castIntFieldToDoubleWithAliasTest() throws IOException { + String query = + "SELECT CAST(age AS DOUBLE) AS test_alias FROM " + TestsConstants.TEST_INDEX_ACCOUNT + + " ORDER BY age LIMIT 5"; + + SearchHit[] hits = query(query).getHits(); + checkSuccessfulFieldCast(hits, "test_alias", "DOUBLE"); + for (int i = 0; i < hits.length; ++i) { + Assert.assertThat(hits[i].getFields().get("test_alias").getValue(), is(20.0)); + } + } + + @Test + public void castIntFieldToStringWithoutAliasTest() throws IOException { + String query = "SELECT CAST(balance AS STRING) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + + " ORDER BY balance LIMIT 1"; + + SearchHit[] hits = query(query).getHits(); + checkSuccessfulFieldCast(hits, "cast_balance", "STRING"); + for (int i = 0; i < hits.length; ++i) { + Assert.assertThat(hits[i].getFields().get("cast_balance").getValue(), is("1011")); + } + } + + @Test + public void castIntFieldToStringWithAliasTest() throws IOException { + String query = "SELECT CAST(balance AS STRING) AS cast_string_alias FROM " + + TestsConstants.TEST_INDEX_ACCOUNT + + " ORDER BY cast_string_alias DESC LIMIT 1"; + + SearchHit[] hits = query(query).getHits(); + checkSuccessfulFieldCast(hits, "cast_string_alias", "STRING"); + for (int i = 0; i < hits.length; ++i) { + Assert.assertThat(hits[i].getFields().get("cast_string_alias").getValue(), is("9838")); + } + + } + + @Test + public void castIntFieldToFloatWithoutAliasJdbcFormatTest() { + JSONObject response = executeJdbcRequest( + "SELECT CAST(balance AS FLOAT) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + + " ORDER BY balance DESC LIMIT 1"); + + verifySchema(response, + schema("cast_balance", null, "float")); + + verifyDataRows(response, + rows(49989)); + } + + @Test + public void castIntFieldToFloatWithAliasJdbcFormatTest() { + JSONObject response = executeJdbcRequest( + "SELECT CAST(balance AS FLOAT) AS jdbc_float_alias " + + "FROM " + TestsConstants.TEST_INDEX_ACCOUNT + " ORDER BY jdbc_float_alias LIMIT 1"); + + verifySchema(response, + schema("jdbc_float_alias", null, "float")); + + verifyDataRows(response, + rows(1011)); + } + + @Test + public void castIntFieldToDoubleWithoutAliasOrderByTest() throws IOException { + String query = "SELECT CAST(age AS DOUBLE) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + + " ORDER BY age LIMIT 1"; + + SearchHit[] hits = query(query).getHits(); + checkSuccessfulFieldCast(hits, "cast_age", "DOUBLE"); + for (int i = 0; i < hits.length; ++i) { + Assert.assertThat(hits[i].getFields().get("cast_age").getValue(), is(20.0)); + } + } + + @Test + public void castIntFieldToDoubleWithAliasOrderByTest() throws IOException { + String query = "SELECT CAST(age AS DOUBLE) AS alias FROM " + TestsConstants.TEST_INDEX_ACCOUNT + + " ORDER BY alias DESC LIMIT 1"; + + SearchHit[] hits = query(query).getHits(); + checkSuccessfulFieldCast(hits, "alias", "DOUBLE"); + for (int i = 0; i < hits.length; ++i) { + Assert.assertThat(hits[i].getFields().get("alias").getValue(), is(40.0)); + } + + } + + @Test + public void castIntFieldToFloatWithoutAliasJdbcFormatGroupByTest() { + JSONObject response = executeJdbcRequest( + "SELECT CAST(balance AS FLOAT) FROM " + + TestsConstants.TEST_INDEX_ACCOUNT + " GROUP BY balance DESC LIMIT 5"); + + verifySchema(response, + schema("CAST(balance AS FLOAT)", null, "float")); + + verifyDataRows(response, + rows(22026), + rows(23285), + rows(36038), + rows(39063), + rows(45493)); + } + + @Test + public void castIntFieldToFloatWithAliasJdbcFormatGroupByTest() { + JSONObject response = executeJdbcRequest( + "SELECT CAST(balance AS FLOAT) AS jdbc_float_alias " + + "FROM " + TestsConstants.TEST_INDEX_ACCOUNT + " GROUP BY jdbc_float_alias ASC LIMIT 5"); + + verifySchema(response, + schema("jdbc_float_alias", "jdbc_float_alias", "float")); + + verifyDataRows(response, + rows("22026.0"), + rows("23285.0"), + rows("36038.0"), + rows("39063.0"), + rows("45493.0")); + } + + @Test + public void castIntFieldToDoubleWithAliasJdbcFormatGroupByTest() { + JSONObject response = executeJdbcRequest( + "SELECT CAST(age AS DOUBLE) AS jdbc_double_alias " + + "FROM " + TestsConstants.TEST_INDEX_ACCOUNT + + " GROUP BY jdbc_double_alias DESC LIMIT 5"); + + verifySchema(response, + schema("jdbc_double_alias", "jdbc_double_alias", "double")); + + verifyDataRows(response, + rows("31.0"), + rows("39.0"), + rows("26.0"), + rows("32.0"), + rows("35.0")); + } + + @Test + public void castKeywordFieldToDatetimeWithoutAliasJdbcFormatTest() { + JSONObject response = executeJdbcRequest("SELECT CAST(date_keyword AS DATETIME) FROM " + + TestsConstants.TEST_INDEX_DATE + " ORDER BY date_keyword"); + + verifySchema(response, schema("cast_date_keyword", null, "date")); + + verifyDataRows(response, + rows("2014-08-19 07:09:13.434"), + rows("2019-09-25 02:04:13.469")); + } + + @Test + public void castKeywordFieldToDatetimeWithAliasJdbcFormatTest() { + JSONObject response = + executeJdbcRequest("SELECT CAST(date_keyword AS DATETIME) AS test_alias FROM " + + TestsConstants.TEST_INDEX_DATE + " ORDER BY date_keyword"); + + verifySchema(response, schema("test_alias", null, "date")); + + verifyDataRows(response, + rows("2014-08-19 07:09:13.434"), + rows("2019-09-25 02:04:13.469")); + } + + @Test + public void castFieldToDatetimeWithWhereClauseJdbcFormatTest() { + JSONObject response = executeJdbcRequest("SELECT CAST(date_keyword AS DATETIME) FROM " + + TestsConstants.TEST_INDEX_DATE + " WHERE date_keyword IS NOT NULL ORDER BY date_keyword"); + + verifySchema(response, schema("cast_date_keyword", null, "date")); + + verifyDataRows(response, + rows("2014-08-19 07:09:13.434"), + rows("2019-09-25 02:04:13.469")); + } + + @Test + public void castFieldToDatetimeWithGroupByJdbcFormatTest() { + JSONObject response = + executeJdbcRequest("SELECT CAST(date_keyword AS DATETIME) AS test_alias FROM " + + TestsConstants.TEST_INDEX_DATE + " GROUP BY test_alias DESC"); + + verifySchema(response, schema("test_alias", "test_alias", "double")); + + verifyDataRows(response, + rows("2014-08-19T07:09:13.434Z"), + rows("2019-09-25T02:04:13.469Z")); + } + + @Test + public void castStatementInWhereClauseGreaterThanTest() { + JSONObject response = executeJdbcRequest("SELECT balance FROM " + TEST_INDEX_ACCOUNT + + " WHERE (account_number < CAST(age AS DOUBLE)) ORDER BY balance LIMIT 5"); + + verifySchema(response, schema("balance", null, "long")); + + verifyDataRows(response, + rows(4180), + rows(5686), + rows(7004), + rows(7831), + rows(14127)); + } + + @Test + public void castStatementInWhereClauseLessThanTest() { + JSONObject response = executeJdbcRequest("SELECT balance FROM " + TEST_INDEX_ACCOUNT + + " WHERE (account_number > CAST(age AS DOUBLE)) ORDER BY balance LIMIT 5"); + + verifySchema(response, schema("balance", null, "long")); + + verifyDataRows(response, + rows(1011), + rows(1031), + rows(1110), + rows(1133), + rows(1172)); + } + + @Test + public void castStatementInWhereClauseEqualToConstantTest() { + JSONObject response = executeJdbcRequest("SELECT balance FROM " + TEST_INDEX_ACCOUNT + + " WHERE (CAST(age AS DOUBLE) = 36.0) ORDER BY balance LIMIT 5"); + + verifySchema(response, schema("balance", null, "long")); + verifyDataRows(response, + rows(1249), + rows(1463), + rows(3960), + rows(5686), + rows(6025)); + } + + @Test + public void castStatementInWhereClauseLessThanConstantTest() { + JSONObject response = executeJdbcRequest("SELECT balance FROM " + TEST_INDEX_ACCOUNT + + " WHERE (CAST(age AS DOUBLE) < 36.0) ORDER BY balance LIMIT 5"); + + verifySchema(response, schema("balance", null, "long")); + + verifyDataRows(response, + rows(1011), + rows(1031), + rows(1110), + rows(1133), + rows(1172)); + } + + /** + * Testing compilation + * Result comparison is empty then comparing different types (Date and keyword) + */ + @Test + public void castStatementInWhereClauseDatetimeCastTest() { + JSONObject response = executeJdbcRequest("SELECT date_keyword FROM " + + TestsConstants.TEST_INDEX_DATE + + " WHERE (CAST(date_keyword AS DATETIME) = \'2014-08-19T07:09:13.434Z\')"); + + String schema_result = "{\"name\":\"date_keyword\",\"type\":\"keyword\"}"; + assertEquals(response.getJSONArray("schema").get(0).toString(), schema_result); + } + + @Test + public void concat_ws_field_and_string() throws Exception { + //here is a bug,csv field with spa + String query = "SELECT " + + " concat_ws('-',age,'-') as age,address from " + + TEST_INDEX_ACCOUNT + " " + + " limit 10 "; + + assertThat( + executeQuery(query), + hitAny(kvString("/fields/age/0", endsWith("--"))) + ); + } + + /** + * Ignore this test case because painless doesn't whitelist String.split function. + * + * @see https://www.elastic.co/guide/en/elasticsearch/painless/7.0/painless-api-reference.html + */ + @Ignore + public void whereConditionLeftFunctionRightVariableEqualTest() throws Exception { + + String query = "SELECT " + + " * from " + + TestsConstants.TEST_INDEX + " " + + " where split(address,' ')[0]='806' limit 1000 "; + + assertThat(executeQuery(query).query("/hits/total"), equalTo(4)); + } + + /** + * Ignore this test case because painless doesn't whitelist String.split function. + * + * @see https://www.elastic.co/guide/en/elasticsearch/painless/7.0/painless-api-reference.html + */ + @Ignore + public void whereConditionLeftFunctionRightVariableGreatTest() throws Exception { + + String query = "SELECT " + + " * from " + + TestsConstants.TEST_INDEX + " " + + " where floor(split(address,' ')[0]+0) > 805 limit 1000 "; + + assertThat(executeQuery(query).query("/hits/total"), equalTo(223)); + } + + @Test + public void concat_ws_fields() throws Exception { + + //here is a bug,csv field with spa + String query = "SELECT " + + " concat_ws('-',age,address) as combine,address from " + + TEST_INDEX_ACCOUNT + " " + + " limit 10 "; + assertThat( + executeQuery(query), + hitAny(kvString("/fields/combine/0", containsString("-"))) + ); + } + + @Test + public void functionLogs() throws Exception { + String query = "SELECT log10(100) as a, log(1) as b, log(2, 4) as c, log2(8) as d from " + + TEST_INDEX_ACCOUNT + " limit 1"; + + assertThat( + executeQuery(query), + hitAny(both(kvDouble("/fields/a/0", equalTo(Math.log10(100)))) + .and(kvDouble("/fields/b/0", equalTo(Math.log(1)))) + .and(kvDouble("/fields/c/0", closeTo(Math.log(4) / Math.log(2), 0.0001))) + .and(kvDouble("/fields/d/0", closeTo(Math.log(8) / Math.log(2), 0.0001)))) + ); + } + + @Test + public void functionPow() throws Exception { + String query = "SELECT pow(account_number, 2) as key," + + "abs(age - 60) as new_age from " + TEST_INDEX_ACCOUNT + + " WHERE firstname = 'Virginia' and lastname='Ayala' limit 1"; + + assertThat( + executeQuery(query), + hitAny(both(kvDouble("/fields/new_age/0", equalTo(21.0))) + .and(kvDouble("/fields/key/0", equalTo(625.0)))) + ); + } + + @Test + public void operatorSubstring() throws IOException { + assertThat( + executeQuery( + "SELECT substring('sampleName', 1, 4) AS substring FROM " + TEST_INDEX_ACCOUNT), + hitAny(kvString("/fields/substring/0", equalTo("samp"))) + ); + + assertThat( + executeQuery( + "SELECT substring('sampleName', 0, 20) AS substring FROM " + TEST_INDEX_ACCOUNT), + hitAny(kvString("/fields/substring/0", equalTo("sampleName"))) + ); + } + + @Test + public void operatorLength() throws IOException { + assertThat( + executeQuery("SELECT LENGTH(lastname) FROM " + TEST_INDEX_ACCOUNT + + " WHERE lastname IS NOT NULL GROUP BY LENGTH(lastname) ORDER BY LENGTH(lastname)", + "jdbc"), + containsString("\"type\": \"integer\"") + ); + + assertThat( + executeQuery("SELECT LENGTH('sampleName') AS length FROM " + TEST_INDEX_ACCOUNT), + hitAny(kvInt("/fields/length/0", equalTo(10))) + ); + + } + + @Test + public void operatorReplace() { + String query = "SELECT REPLACE('elastic', 'el', 'fant') FROM " + TEST_INDEX_ACCOUNT; + assertThat( + executeQuery(query, "jdbc"), + containsString("fantastic") + ); + } + + @Test + public void operatorLocate() throws IOException { + String query = "SELECT LOCATE('a', lastname, 0) FROM " + TEST_INDEX_ACCOUNT + + + " WHERE lastname IS NOT NULL GROUP BY LOCATE('a', lastname, 0) ORDER BY LOCATE('a', lastname, 0)"; + assertThat( + executeQuery(query, "jdbc"), containsString("\"type\": \"integer\"") + ); + + assertThat( + executeQuery("SELECT LOCATE('a', 'sampleName', 3) AS locate FROM " + TEST_INDEX_ACCOUNT), + hitAny(kvInt("/fields/locate/0", equalTo(8))) + ); + assertThat( + executeQuery("SELECT LOCATE('a', 'sampleName') AS locate FROM " + TEST_INDEX_ACCOUNT), + hitAny(kvInt("/fields/locate/0", equalTo(2))) + ); + } + + @Test + public void rtrim() throws IOException { + assertThat( + executeQuery("SELECT RTRIM(' sampleName ') AS rtrim FROM " + TEST_INDEX_ACCOUNT), + hitAny(kvString("/fields/rtrim/0", equalTo(" sampleName"))) + ); + } + + @Test + public void ltrim() throws IOException { + assertThat( + executeQuery("SELECT LTRIM(' sampleName ') AS ltrim FROM " + TEST_INDEX_ACCOUNT), + hitAny(kvString("/fields/ltrim/0", equalTo("sampleName "))) + ); + } + + @Test + public void ascii() throws IOException { + assertThat( + executeQuery("SELECT ASCII(lastname) FROM " + TEST_INDEX_ACCOUNT + + + " WHERE lastname IS NOT NULL GROUP BY ASCII(lastname) ORDER BY ASCII(lastname) LIMIT 5", + "jdbc"), + containsString("\"type\": \"integer\"") + ); + assertThat( + executeQuery("SELECT ASCII('sampleName') AS ascii FROM " + TEST_INDEX_ACCOUNT), + hitAny(kvInt("/fields/ascii/0", equalTo(115))) + ); + } + + /** + * The following tests for LEFT and RIGHT are ignored because the ES client fails to parse "LEFT"/"RIGHT" in + * the integTest + */ + @Ignore + @Test + public void left() throws IOException { + assertThat( + executeQuery( + "SELECT LEFT('sample', 2) AS left FROM " + TEST_INDEX_ACCOUNT + " ORDER BY left"), + hitAny(kvString("/fields/left/0", equalTo("sa"))) + ); + assertThat( + executeQuery( + "SELECT LEFT('sample', 20) AS left FROM " + TEST_INDEX_ACCOUNT + " ORDER BY left"), + hitAny(kvString("/fields/left/0", equalTo("sample"))) + ); + } + + @Ignore + @Test + public void right() throws IOException { + assertThat( + executeQuery( + "SELECT RIGHT('elastic', 3) AS right FROM " + TEST_INDEX_ACCOUNT + " ORDER BY right"), + hitAny(kvString("/fields/right/0", equalTo("tic"))) + ); + assertThat( + executeQuery( + "SELECT RIGHT('elastic', 20) AS right FROM " + TEST_INDEX_ACCOUNT + " ORDER BY right"), + hitAny(kvString("/fields/right/0", equalTo("elastic"))) + ); + } + + @Test + public void ifFuncShouldPassJDBC() { + JSONObject response = executeJdbcRequest( + "SELECT IF(age > 30, 'True', 'False') AS Ages FROM " + TEST_INDEX_ACCOUNT + + " WHERE age IS NOT NULL GROUP BY Ages"); + assertEquals("Ages", response.query("/schema/0/name")); + assertEquals("Ages", response.query("/schema/0/alias")); + assertEquals("double", response.query("/schema/0/type")); + } + + @Test + public void ifFuncWithBinaryComparisonAsConditionTest() throws IOException { + assertThat( + executeQuery("SELECT IF(2 > 0, 'hello', 'world') AS ifTrue FROM " + TEST_INDEX_ACCOUNT), + hitAny(kvString("/fields/ifTrue/0", equalTo("hello"))) + ); + assertThat( + executeQuery("SELECT IF(2 = 0, 'hello', 'world') AS ifFalse FROM " + TEST_INDEX_ACCOUNT), + hitAny(kvString("/fields/ifFalse/0", equalTo("world"))) + ); + } + + @Test + public void ifFuncWithBooleanExprInputAsConditionTest() throws IOException { + assertThat( + executeQuery("SELECT IF(true, 1, 0) AS ifBoolean FROM " + TEST_INDEX_ACCOUNT), + hitAny(kvInt("/fields/ifBoolean/0", equalTo(1))) + ); + } + + @Test + public void ifFuncWithNullInputAsConditionTest() throws IOException { + assertThat( + executeQuery("SELECT IF(null, 1, 0) AS ifNull FROM " + TEST_INDEX_ACCOUNT), + hitAny(kvInt("/fields/ifNull/0", equalTo(0))) + ); + } + + @Test + public void ifnullShouldPassJDBC() throws IOException { + JSONObject response = executeJdbcRequest( + "SELECT IFNULL(lastname, 'unknown') AS name FROM " + TEST_INDEX_ACCOUNT + + " GROUP BY name"); + assertEquals("name", response.query("/schema/0/name")); + assertEquals("name", response.query("/schema/0/alias")); + assertEquals("double", response.query("/schema/0/type")); + } + + @Test + public void ifnullWithNotNullInputTest() throws IOException { + assertThat( + executeQuery("SELECT IFNULL('sample', 'IsNull') AS ifnull FROM " + TEST_INDEX_ACCOUNT), + hitAny(kvString("/fields/ifnull/0", equalTo("sample"))) + ); + } + + @Test + public void ifnullWithNullInputTest() throws IOException { + assertThat( + executeQuery("SELECT IFNULL(null, 10) AS ifnull FROM " + TEST_INDEX_ACCOUNT), + hitAny(kvInt("/fields/ifnull/0", equalTo(10))) + ); + assertThat( + executeQuery("SELECT IFNULL('', 10) AS ifnull FROM " + TEST_INDEX_ACCOUNT), + hitAny(kvString("/fields/ifnull/0", equalTo(""))) + ); + } + + @Test + public void isnullShouldPassJDBC() { + JSONObject response = + executeJdbcRequest( + "SELECT ISNULL(lastname) AS name FROM " + TEST_INDEX_ACCOUNT + " GROUP BY name"); + assertEquals("name", response.query("/schema/0/name")); + assertEquals("name", response.query("/schema/0/alias")); + assertEquals("integer", response.query("/schema/0/type")); + } + + @Test + public void isnullWithNotNullInputTest() throws IOException { + assertThat( + executeQuery("SELECT ISNULL('elastic') AS isnull FROM " + TEST_INDEX_ACCOUNT), + hitAny(kvInt("/fields/isnull/0", equalTo(0))) + ); + assertThat( + executeQuery("SELECT ISNULL('') AS isnull FROM " + TEST_INDEX_ACCOUNT), + hitAny(kvInt("/fields/isnull/0", equalTo(0))) + ); + } + + @Test + public void isnullWithNullInputTest() throws IOException { + assertThat( + executeQuery("SELECT ISNULL(null) AS isnull FROM " + TEST_INDEX_ACCOUNT), + hitAny(kvInt("/fields/isnull/0", equalTo(1))) + ); + } + + @Test + public void isnullWithMathExpr() throws IOException { + assertThat( + executeQuery("SELECT ISNULL(1+1) AS isnull FROM " + TEST_INDEX_ACCOUNT), + hitAny(kvInt("/fields/isnull/0", equalTo(0))) + ); + assertThat( + executeQuery("SELECT ISNULL(1+1*1/0) AS isnull FROM " + TEST_INDEX_ACCOUNT), + hitAny(kvInt("/fields/isnull/0", equalTo(1))) + ); + } + + /** + * Ignore this test case because painless doesn't whitelist String.split function. + * + * @see https://www.elastic.co/guide/en/elasticsearch/painless/7.0/painless-api-reference.html + */ + @Ignore + public void split_field() throws Exception { + + //here is a bug,csv field with spa + String query = "SELECT " + + " split(address,' ')[0],age from " + + TestsConstants.TEST_INDEX + " where address is not null " + + " limit 10 "; + } + + @Test + public void literal() throws Exception { + String query = "SELECT 10 " + + "from " + TEST_INDEX_ACCOUNT + " limit 1"; + final SearchHit[] hits = query(query).getHits(); + assertThat(hits[0].getFields(), hasValue(contains(10))); + } + + @Test + public void literalWithDoubleValue() throws Exception { + String query = "SELECT 10.0 " + + "from " + TEST_INDEX_ACCOUNT + " limit 1"; + + final SearchHit[] hits = query(query).getHits(); + assertThat(hits[0].getFields(), hasValue(contains(10.0))); + } + + @Test + public void literalWithAlias() throws Exception { + String query = "SELECT 10 as key " + + "from " + TEST_INDEX_ACCOUNT + " limit 1"; + final SearchHit[] hits = query(query).getHits(); + + assertThat(hits.length, is(1)); + assertThat(hits[0].getFields(), hasEntry(is("key"), contains(10))); + } + + @Test + public void literalMultiField() throws Exception { + String query = "SELECT 1, 2 " + + "from " + TEST_INDEX_ACCOUNT + " limit 1"; + final SearchHit[] hits = query(query).getHits(); + + assertThat(hits.length, is(1)); + assertThat(hits[0].getFields(), allOf(hasValue(contains(1)), hasValue(contains(2)))); + } + + private SearchHits query(String query) throws IOException { + final String rsp = executeQueryWithStringOutput(query); + + final XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser( + NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, + rsp); + return SearchResponse.fromXContent(parser).getHits(); + } + + private void checkSuccessfulFieldCast(SearchHit[] hits, String field, String castType) { + for (int i = 0; i < hits.length; ++i) { + Assert.assertThat(hits[i].getFields(), IsMapContaining.hasKey(field)); + switch (castType) { + case "FLOAT": + assertTrue(hits[i].getFields().get(field).getValue() instanceof Float); + break; + case "DOUBLE": + assertTrue(hits[i].getFields().get(field).getValue() instanceof Double); + break; + case "INT": + assertTrue(hits[i].getFields().get(field).getValue() instanceof Integer); + break; + case "STRING": + assertTrue(hits[i].getFields().get(field).getValue() instanceof String); + break; + case "DATETIME": + assertTrue(hits[i].getFields().get(field).getValue() instanceof Date); + break; + case "LONG": + assertTrue(hits[i].getFields().get(field).getValue() instanceof Long); + break; + } + } + } + + private JSONObject executeJdbcRequest(String query) { + return new JSONObject(executeQuery(query, "jdbc")); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLIntegTestCase.java new file mode 100644 index 0000000000..3981aa99d0 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLIntegTestCase.java @@ -0,0 +1,544 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.createIndexByRestClient; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getAccountIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getBankIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getBankWithNullValuesIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getDateIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getDateTimeIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getDogIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getDogs2IndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getDogs3IndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getEmployeeNestedTypeIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getGameOfThronesIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getJoinTypeIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getLocationIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getNestedSimpleIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getNestedTypeIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getOdbcIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getOrderIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getPeople2IndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getPhraseIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getResponseBody; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getWeblogsIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.isIndexExist; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.loadDataByRestClient; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlAction.CURSOR_CLOSE_ENDPOINT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlAction.EXPLAIN_API_ENDPOINT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; + +import com.google.common.base.Strings; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Locale; +import javax.management.MBeanServerInvocationHandler; +import javax.management.ObjectName; +import javax.management.remote.JMXConnector; +import javax.management.remote.JMXConnectorFactory; +import javax.management.remote.JMXServiceURL; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; + +/** + * ES Rest integration test base for SQL testing + */ +public abstract class SQLIntegTestCase extends ODFERestTestCase { + + public static final String PERSISTENT = "persistent"; + public static final String TRANSIENT = "transient"; + + @Before + public void setUpIndices() throws Exception { + if (client() == null) { + initClient(); + } + + increaseScriptMaxCompilationsRate(); + init(); + } + + @Override + protected boolean preserveClusterUponCompletion() { + return true; // Preserve test index, template and settings between test cases + } + + /** + * We need to be able to dump the jacoco coverage before cluster is shut down. + * The new internal testing framework removed some of the gradle tasks we were listening to + * to choose a good time to do it. This will dump the executionData to file after each test. + * TODO: This is also currently just overwriting integTest.exec with the updated execData without + * resetting after writing each time. This can be improved to either write an exec file per test + * or by letting jacoco append to the file + */ + public interface IProxy { + byte[] getExecutionData(boolean reset); + + void dump(boolean reset); + + void reset(); + } + + @AfterClass + public static void dumpCoverage() { + // jacoco.dir is set in sqlplugin-coverage.gradle, if it doesn't exist we don't + // want to collect coverage so we can return early + String jacocoBuildPath = System.getProperty("jacoco.dir"); + if (Strings.isNullOrEmpty(jacocoBuildPath)) { + return; + } + + String serverUrl = "service:jmx:rmi:///jndi/rmi://127.0.0.1:7777/jmxrmi"; + try (JMXConnector connector = JMXConnectorFactory.connect(new JMXServiceURL(serverUrl))) { + IProxy proxy = MBeanServerInvocationHandler.newProxyInstance( + connector.getMBeanServerConnection(), new ObjectName("org.jacoco:type=Runtime"), + IProxy.class, + false); + + Path path = Paths.get(jacocoBuildPath + "/integTest.exec"); + Files.write(path, proxy.getExecutionData(false)); + } catch (Exception ex) { + throw new RuntimeException("Failed to dump coverage", ex); + } + } + + /** + * As JUnit JavaDoc says: + * "The @AfterClass methods declared in superclasses will be run after those of the current class." + * So this method is supposed to run before closeClients() in parent class. + */ + @AfterClass + public static void cleanUpIndices() throws IOException { + wipeAllODFEIndices(); + wipeAllClusterSettings(); + } + + /** + * Increase script.max_compilations_rate to large enough, which is only 75/5min by default. + * This issue is due to our painless script not using params passed to compiled script. + */ + private void increaseScriptMaxCompilationsRate() throws IOException { + updateClusterSettings( + new ClusterSetting("transient", "script.max_compilations_rate", "10000/1m")); + } + + protected static void wipeAllClusterSettings() throws IOException { + updateClusterSettings(new ClusterSetting("persistent", "*", null)); + updateClusterSettings(new ClusterSetting("transient", "*", null)); + } + + /** + * Provide for each test to load test index, data and other setup work + */ + protected void init() throws Exception { + } + + /** + * Make it thread-safe in case tests are running in parallel but does not guarantee + * if test like DeleteIT that mutates cluster running in parallel. + */ + protected synchronized void loadIndex(Index index) throws IOException { + String indexName = index.getName(); + String mapping = index.getMapping(); + String dataSet = index.getDataSet(); + + if (!isIndexExist(client(), indexName)) { + createIndexByRestClient(client(), indexName, mapping); + loadDataByRestClient(client(), indexName, dataSet); + } + } + + protected Request getSqlRequest(String request, boolean explain) { + return getSqlRequest(request, explain, "json"); + } + + protected Request getSqlRequest(String request, boolean explain, String requestType) { + String queryEndpoint = String.format("%s?format=%s", QUERY_API_ENDPOINT, requestType); + Request sqlRequest = new Request("POST", explain ? EXPLAIN_API_ENDPOINT : queryEndpoint); + sqlRequest.setJsonEntity(request); + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + sqlRequest.setOptions(restOptionsBuilder); + + return sqlRequest; + } + + protected Request getSqlCursorCloseRequest(String cursorRequest) { + String queryEndpoint = String.format("%s?format=%s", CURSOR_CLOSE_ENDPOINT, "jdbc"); + Request sqlRequest = new Request("POST", queryEndpoint); + sqlRequest.setJsonEntity(cursorRequest); + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + sqlRequest.setOptions(restOptionsBuilder); + + return sqlRequest; + } + + protected String executeQuery(String query, String requestType) { + try { + String endpoint = "/_opendistro/_sql?format=" + requestType; + String requestBody = makeRequest(query); + + Request sqlRequest = new Request("POST", endpoint); + sqlRequest.setJsonEntity(requestBody); + + Response response = client().performRequest(sqlRequest); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + String responseString = getResponseBody(response, true); + + return responseString; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + protected String executeFetchQuery(String query, int fetchSize, String requestType) + throws IOException { + String endpoint = "/_opendistro/_sql?format=" + requestType; + String requestBody = makeRequest(query, fetchSize); + + Request sqlRequest = new Request("POST", endpoint); + sqlRequest.setJsonEntity(requestBody); + + Response response = client().performRequest(sqlRequest); + String responseString = getResponseBody(response, true); + return responseString; + } + + protected String executeFetchLessQuery(String query, String requestType) throws IOException { + + String endpoint = "/_opendistro/_sql?format=" + requestType; + String requestBody = makeFetchLessRequest(query); + + Request sqlRequest = new Request("POST", endpoint); + sqlRequest.setJsonEntity(requestBody); + + Response response = client().performRequest(sqlRequest); + String responseString = getResponseBody(response, true); + return responseString; + } + + protected Request buildGetEndpointRequest(final String sqlQuery) { + + final String utf8CharsetName = StandardCharsets.UTF_8.name(); + String urlEncodedQuery = ""; + + try { + urlEncodedQuery = URLEncoder.encode(sqlQuery, utf8CharsetName); + } catch (UnsupportedEncodingException e) { + // Will never reach here since UTF-8 is always supported + Assert.fail(utf8CharsetName + " not available"); + } + + final String requestUrl = String.format(Locale.ROOT, "%s?sql=%s&format=%s", QUERY_API_ENDPOINT, + urlEncodedQuery, "json"); + return new Request("GET", requestUrl); + } + + protected JSONObject executeQuery(final String sqlQuery) throws IOException { + + final String requestBody = makeRequest(sqlQuery); + return executeRequest(requestBody); + } + + protected String explainQuery(final String sqlQuery) throws IOException { + + final String requestBody = makeRequest(sqlQuery); + return executeExplainRequest(requestBody); + } + + protected String executeQueryWithStringOutput(final String sqlQuery) throws IOException { + + final String requestString = makeRequest(sqlQuery); + return executeRequest(requestString, false); + } + + protected JSONObject executeRequest(final String requestBody) throws IOException { + + return new JSONObject(executeRequest(requestBody, false)); + } + + protected String executeExplainRequest(final String requestBody) throws IOException { + + return executeRequest(requestBody, true); + } + + private String executeRequest(final String requestBody, final boolean isExplainQuery) + throws IOException { + + Request sqlRequest = getSqlRequest(requestBody, isExplainQuery); + return executeRequest(sqlRequest); + } + + protected static String executeRequest(final Request request) throws IOException { + + Response response = client().performRequest(request); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + return getResponseBody(response); + } + + protected JSONObject executeQueryWithGetRequest(final String sqlQuery) throws IOException { + + final Request request = buildGetEndpointRequest(sqlQuery); + final String result = executeRequest(request); + return new JSONObject(result); + } + + protected JSONObject executeCursorQuery(final String cursor) throws IOException { + final String requestBody = makeCursorRequest(cursor); + Request sqlRequest = getSqlRequest(requestBody, false, "jdbc"); + return new JSONObject(executeRequest(sqlRequest)); + } + + protected JSONObject executeCursorCloseQuery(final String cursor) throws IOException { + final String requestBody = makeCursorRequest(cursor); + Request sqlRequest = getSqlCursorCloseRequest(requestBody); + return new JSONObject(executeRequest(sqlRequest)); + } + + protected static JSONObject updateClusterSettings(ClusterSetting setting) throws IOException { + Request request = new Request("PUT", "/_cluster/settings"); + String persistentSetting = String.format(Locale.ROOT, + "{\"%s\": {\"%s\": %s}}", setting.type, setting.name, setting.value); + request.setJsonEntity(persistentSetting); + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + request.setOptions(restOptionsBuilder); + return new JSONObject(executeRequest(request)); + } + + protected static JSONObject getAllClusterSettings() throws IOException { + Request request = new Request("GET", "/_cluster/settings?flat_settings&include_defaults"); + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + request.setOptions(restOptionsBuilder); + return new JSONObject(executeRequest(request)); + } + + protected static class ClusterSetting { + private final String type; + private final String name; + private final String value; + + public ClusterSetting(String type, String name, String value) { + this.type = type; + this.name = name; + this.value = (value == null) ? "null" : ("\"" + value + "\""); + } + + ClusterSetting nullify() { + return new ClusterSetting(type, name, null); + } + + @Override + public String toString() { + return "ClusterSetting{" + + "type='" + type + '\'' + + ", path='" + name + '\'' + + ", value='" + value + '\'' + + '}'; + } + } + + protected String makeRequest(String query) { + return makeRequest(query, 0); + } + + protected String makeRequest(String query, int fetch_size) { + return String.format("{\n" + + " \"fetch_size\": \"%s\",\n" + + " \"query\": \"%s\"\n" + + "}", fetch_size, query); + } + + protected String makeFetchLessRequest(String query) { + return String.format("{\n" + + " \"query\": \"%s\"\n" + + "}", query); + } + + protected String makeCursorRequest(String cursor) { + return String.format("{\"cursor\":\"%s\"}", cursor); + } + + protected JSONArray getHits(JSONObject response) { + Assert.assertTrue(response.getJSONObject("hits").has("hits")); + + return response.getJSONObject("hits").getJSONArray("hits"); + } + + protected int getTotalHits(JSONObject response) { + Assert.assertTrue(response.getJSONObject("hits").has("total")); + Assert.assertTrue(response.getJSONObject("hits").getJSONObject("total").has("value")); + + return response.getJSONObject("hits").getJSONObject("total").getInt("value"); + } + + protected JSONObject getSource(JSONObject hit) { + return hit.getJSONObject("_source"); + } + + /** + * Enum for associating test index with relevant mapping and data. + */ + public enum Index { + ONLINE(TestsConstants.TEST_INDEX_ONLINE, + "online", + null, + "src/test/resources/online.json"), + ACCOUNT(TestsConstants.TEST_INDEX_ACCOUNT, + "account", + getAccountIndexMapping(), + "src/test/resources/accounts.json"), + PHRASE(TestsConstants.TEST_INDEX_PHRASE, + "phrase", + getPhraseIndexMapping(), + "src/test/resources/phrases.json"), + DOG(TestsConstants.TEST_INDEX_DOG, + "dog", + getDogIndexMapping(), + "src/test/resources/dogs.json"), + DOGS2(TestsConstants.TEST_INDEX_DOG2, + "dog", + getDogs2IndexMapping(), + "src/test/resources/dogs2.json"), + DOGS3(TestsConstants.TEST_INDEX_DOG3, + "dog", + getDogs3IndexMapping(), + "src/test/resources/dogs3.json"), + DOGSSUBQUERY(TestsConstants.TEST_INDEX_DOGSUBQUERY, + "dog", + getDogIndexMapping(), + "src/test/resources/dogsubquery.json"), + PEOPLE(TestsConstants.TEST_INDEX_PEOPLE, + "people", + null, + "src/test/resources/peoples.json"), + PEOPLE2(TestsConstants.TEST_INDEX_PEOPLE2, + "people", + getPeople2IndexMapping(), + "src/test/resources/people2.json"), + GAME_OF_THRONES(TestsConstants.TEST_INDEX_GAME_OF_THRONES, + "gotCharacters", + getGameOfThronesIndexMapping(), + "src/test/resources/game_of_thrones_complex.json"), + SYSTEM(TestsConstants.TEST_INDEX_SYSTEM, + "systems", + null, + "src/test/resources/systems.json"), + ODBC(TestsConstants.TEST_INDEX_ODBC, + "odbc", + getOdbcIndexMapping(), + "src/test/resources/odbc-date-formats.json"), + LOCATION(TestsConstants.TEST_INDEX_LOCATION, + "location", + getLocationIndexMapping(), + "src/test/resources/locations.json"), + LOCATION_TWO(TestsConstants.TEST_INDEX_LOCATION2, + "location2", + getLocationIndexMapping(), + "src/test/resources/locations2.json"), + NESTED(TestsConstants.TEST_INDEX_NESTED_TYPE, + "nestedType", + getNestedTypeIndexMapping(), + "src/test/resources/nested_objects.json"), + NESTED_WITH_QUOTES(TestsConstants.TEST_INDEX_NESTED_WITH_QUOTES, + "nestedType", + getNestedTypeIndexMapping(), + "src/test/resources/nested_objects_quotes_in_values.json"), + EMPLOYEE_NESTED(TestsConstants.TEST_INDEX_EMPLOYEE_NESTED, + "_doc", + getEmployeeNestedTypeIndexMapping(), + "src/test/resources/employee_nested.json"), + JOIN(TestsConstants.TEST_INDEX_JOIN_TYPE, + "joinType", + getJoinTypeIndexMapping(), + "src/test/resources/join_objects.json"), + BANK(TestsConstants.TEST_INDEX_BANK, + "account", + getBankIndexMapping(), + "src/test/resources/bank.json"), + BANK_TWO(TestsConstants.TEST_INDEX_BANK_TWO, + "account_two", + getBankIndexMapping(), + "src/test/resources/bank_two.json"), + BANK_WITH_NULL_VALUES(TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES, + "account_null", + getBankWithNullValuesIndexMapping(), + "src/test/resources/bank_with_null_values.json"), + ORDER(TestsConstants.TEST_INDEX_ORDER, + "_doc", + getOrderIndexMapping(), + "src/test/resources/order.json"), + WEBLOG(TestsConstants.TEST_INDEX_WEBLOG, + "weblog", + getWeblogsIndexMapping(), + "src/test/resources/weblogs.json"), + DATE(TestsConstants.TEST_INDEX_DATE, + "dates", + getDateIndexMapping(), + "src/test/resources/dates.json"), + DATETIME(TestsConstants.TEST_INDEX_DATE_TIME, + "_doc", + getDateTimeIndexMapping(), + "src/test/resources/datetime.json"), + NESTED_SIMPLE(TestsConstants.TEST_INDEX_NESTED_SIMPLE, + "_doc", + getNestedSimpleIndexMapping(), + "src/test/resources/nested_simple.json"); + + private final String name; + private final String type; + private final String mapping; + private final String dataSet; + + Index(String name, String type, String mapping, String dataSet) { + this.name = name; + this.type = type; + this.mapping = mapping; + this.dataSet = dataSet; + } + + public String getName() { + return this.name; + } + + public String getType() { + return this.type; + } + + public String getMapping() { + return this.mapping; + } + + public String getDataSet() { + return this.dataSet; + } + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/ShowIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/ShowIT.java new file mode 100644 index 0000000000..a60e4cb077 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/ShowIT.java @@ -0,0 +1,84 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.createIndexByRestClient; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.isIndexExist; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.Assert; +import org.junit.Test; + +public class ShowIT extends SQLIntegTestCase { + + @Override + protected void init() { + // Note: not using the existing TEST_INDEX_* indices, since underscore in the names causes issues + createEmptyIndexIfNotExist("abcdefg"); + createEmptyIndexIfNotExist("abcdefghijk"); + createEmptyIndexIfNotExist("abcdijk"); + } + + @Test + public void showAllMatchAll() throws IOException { + + showIndexTest("%", 3, false); + } + + @Test + public void showIndexMatchPrefix() throws IOException { + + showIndexTest("abcdefg" + "%", 2, true); + } + + @Test + public void showIndexMatchSuffix() throws IOException { + + showIndexTest("%ijk", 2, true); + } + + @Test + public void showIndexMatchExact() throws IOException { + + showIndexTest("abcdefg", 1, true); + } + + private void showIndexTest(String querySuffix, int expectedMatches, boolean exactMatch) + throws IOException { + + final String query = "SHOW TABLES LIKE " + querySuffix; + JSONObject result = executeQuery(query); + + if (exactMatch) { + Assert.assertThat(result.length(), equalTo(expectedMatches)); + } else { + Assert.assertThat(result.length(), greaterThanOrEqualTo(expectedMatches)); + } + for (String indexName : result.keySet()) { + Assert.assertTrue(result.getJSONObject(indexName).has("mappings")); + } + } + + private void createEmptyIndexIfNotExist(String indexName) { + if (!isIndexExist(client(), indexName)) { + createIndexByRestClient(client(), indexName, null); + } + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SourceFieldIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SourceFieldIT.java new file mode 100644 index 0000000000..ebeb854a74 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SourceFieldIT.java @@ -0,0 +1,101 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; + +import java.io.IOException; +import java.util.Set; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.json.JSONObject; +import org.junit.Assert; +import org.junit.Test; + +public class SourceFieldIT extends SQLIntegTestCase { + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + } + + @Test + public void includeTest() throws IOException { + SearchHits response = query(String.format( + "SELECT include('*name','*ge'),include('b*'),include('*ddre*'),include('gender') FROM %s/account LIMIT 1000", + TEST_INDEX_ACCOUNT)); + for (SearchHit hit : response.getHits()) { + Set keySet = hit.getSourceAsMap().keySet(); + for (String field : keySet) { + Assert.assertTrue(field.endsWith("name") || field.endsWith("ge") || field.startsWith("b") || + field.contains("ddre") || field.equals("gender")); + } + } + + } + + @Test + public void excludeTest() throws IOException { + + SearchHits response = query(String.format( + "SELECT exclude('*name','*ge'),exclude('b*'),exclude('*ddre*'),exclude('gender') FROM %s/account LIMIT 1000", + TEST_INDEX_ACCOUNT)); + + for (SearchHit hit : response.getHits()) { + Set keySet = hit.getSourceAsMap().keySet(); + for (String field : keySet) { + Assert.assertFalse( + field.endsWith("name") || field.endsWith("ge") || field.startsWith("b") || + field.contains("ddre") || field.equals("gender")); + } + } + } + + @Test + public void allTest() throws IOException { + + SearchHits response = query(String.format( + "SELECT exclude('*name','*ge'),include('b*'),exclude('*ddre*'),include('gender') FROM %s/account LIMIT 1000", + TEST_INDEX_ACCOUNT)); + + for (SearchHit hit : response.getHits()) { + Set keySet = hit.getSourceAsMap().keySet(); + for (String field : keySet) { + Assert + .assertFalse(field.endsWith("name") || field.endsWith("ge") || field.contains("ddre")); + Assert.assertTrue(field.startsWith("b") || field.equals("gender")); + } + } + } + + private SearchHits query(String query) throws IOException { + final JSONObject jsonObject = executeQuery(query); + + final XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser( + NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, + jsonObject.toString()); + return SearchResponse.fromXContent(parser).getHits(); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SubqueryIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SubqueryIT.java new file mode 100644 index 0000000000..c1a08bb00e --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SubqueryIT.java @@ -0,0 +1,382 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_DOGSUBQUERY; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_EMPLOYEE_NESTED; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.hitAll; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.kvInt; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.kvString; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.rows; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.schema; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyDataRows; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifySchema; +import static org.hamcrest.Matchers.both; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.core.Is.is; + +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; +import com.google.common.collect.Ordering; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import org.elasticsearch.client.ResponseException; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +public class SubqueryIT extends SQLIntegTestCase { + + @Rule + public ExpectedException exceptionRule = ExpectedException.none(); + + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + loadIndex(Index.DOGSSUBQUERY); + loadIndex(Index.EMPLOYEE_NESTED); + } + + @Test + public void testIN() throws IOException { + String query = String.format(Locale.ROOT, + "SELECT dog_name " + + "FROM %s A " + + "WHERE holdersName IN (SELECT firstname FROM %s B) " + + "AND dog_name <> 'babala'", + TEST_INDEX_DOGSUBQUERY, TEST_INDEX_ACCOUNT); + + JSONObject response = executeQuery(query); + assertThat( + response, + hitAll( + kvString("/_source/A.dog_name", is("snoopy")), + kvString("/_source/A.dog_name", is("gogo")) + ) + ); + } + + @Test + public void testINWithAlias() throws IOException { + String query = String.format(Locale.ROOT, + "SELECT A.dog_name " + + "FROM %s A " + + "WHERE A.holdersName IN (SELECT B.firstname FROM %s B) " + + "AND A.dog_name <> 'babala'", + TEST_INDEX_DOGSUBQUERY, TEST_INDEX_ACCOUNT); + + JSONObject response = executeQuery(query); + assertThat( + response, + hitAll( + kvString("/_source/A.dog_name", is("snoopy")), + kvString("/_source/A.dog_name", is("gogo")) + ) + ); + } + + @Test + public void testINSelectAll() throws IOException { + String query = String.format(Locale.ROOT, + "SELECT * " + + "FROM %s A " + + "WHERE holdersName IN (SELECT firstname FROM %s B) " + + "AND dog_name <> 'babala'", + TEST_INDEX_DOGSUBQUERY, TEST_INDEX_ACCOUNT); + + JSONObject response = executeQuery(query); + assertThat( + response, + hitAll( + both(kvString("/_source/A.dog_name", is("snoopy"))) + .and(kvString("/_source/A.holdersName", is("Hattie"))) + .and(kvInt("/_source/A.age", is(4))), + both(kvString("/_source/A.dog_name", is("gogo"))) + .and(kvString("/_source/A.holdersName", is("Gabrielle"))) + .and(kvInt("/_source/A.age", is(6))) + ) + ); + } + + @Test + public void testINWithInnerWhere() throws IOException { + String query = String.format(Locale.ROOT, + "SELECT dog_name " + + "FROM %s A " + + "WHERE holdersName IN (SELECT firstname FROM %s B WHERE age <> 36) " + + "AND dog_name <> 'babala'", + TEST_INDEX_DOGSUBQUERY, TEST_INDEX_ACCOUNT); + + JSONObject response = executeQuery(query); + assertThat( + response, + hitAll( + kvString("/_source/A.dog_name", is("gogo")) + ) + ); + } + + @Test + public void testNotSupportedQuery() throws IOException { + exceptionRule.expect(ResponseException.class); + exceptionRule.expectMessage("Unsupported subquery"); + String query = String.format(Locale.ROOT, + "SELECT dog_name " + + "FROM %s A " + + "WHERE holdersName NOT IN (SELECT firstname FROM %s B WHERE age <> 36) " + + "AND dog_name <> 'babala'", + TEST_INDEX_DOGSUBQUERY, TEST_INDEX_ACCOUNT); + executeQuery(query); + } + + // todo Pending on DISTINCT support in JOIN + @Ignore + @Test + public void testINWithDuplicate() throws IOException { + String query = String.format(Locale.ROOT, + "SELECT dog_name " + + "FROM %s A " + + "WHERE holdersName IN (SELECT firstname FROM %s B)", + TEST_INDEX_DOGSUBQUERY, TEST_INDEX_ACCOUNT); + + JSONObject response = executeQuery(query); + assertThat( + response, + hitAll( + kvString("/_source/A.dog_name", is("snoopy")), + kvString("/_source/A.dog_name", is("babala")) + ) + ); + } + + @Test + public void nonCorrelatedExists() throws IOException { + String query = String.format(Locale.ROOT, + "SELECT e.name " + + "FROM %s as e " + + "WHERE EXISTS (SELECT * FROM e.projects as p)", + TEST_INDEX_EMPLOYEE_NESTED); + + JSONObject response = executeQuery(query); + assertThat( + response, + hitAll( + kvString("/_source/name", is("Bob Smith")), + kvString("/_source/name", is("Jane Smith")) + ) + ); + } + + @Test + public void nonCorrelatedExistsWhere() throws IOException { + String query = String.format(Locale.ROOT, + "SELECT e.name " + + "FROM %s as e " + + "WHERE EXISTS (SELECT * FROM e.projects as p WHERE p.name LIKE 'aurora')", + TEST_INDEX_EMPLOYEE_NESTED); + + JSONObject response = executeQuery(query); + assertThat( + response, + hitAll( + kvString("/_source/name", is("Bob Smith")) + ) + ); + } + + @Test + public void nonCorrelatedExistsParentWhere() throws IOException { + String query = String.format(Locale.ROOT, + "SELECT e.name " + + "FROM %s as e " + + "WHERE EXISTS (SELECT * FROM e.projects as p WHERE p.name LIKE 'security') " + + "AND e.name LIKE 'jane'", + TEST_INDEX_EMPLOYEE_NESTED); + + JSONObject response = executeQuery(query); + assertThat( + response, + hitAll( + kvString("/_source/name", is("Jane Smith")) + ) + ); + } + + @Test + public void nonCorrelatedNotExists() throws IOException { + String query = String.format(Locale.ROOT, + "SELECT e.name " + + "FROM %s as e " + + "WHERE NOT EXISTS (SELECT * FROM e.projects as p)", + TEST_INDEX_EMPLOYEE_NESTED); + + JSONObject response = executeQuery(query); + assertThat( + response, + hitAll( + kvString("/_source/name", is("Susan Smith")), + kvString("/_source/name", is("John Doe")) + ) + ); + } + + @Test + public void nonCorrelatedNotExistsWhere() throws IOException { + String query = String.format(Locale.ROOT, + "SELECT e.name " + + "FROM %s as e " + + "WHERE NOT EXISTS (SELECT * FROM e.projects as p WHERE p.name LIKE 'aurora')", + TEST_INDEX_EMPLOYEE_NESTED); + + JSONObject response = executeQuery(query); + assertThat( + response, + hitAll( + kvString("/_source/name", is("Susan Smith")), + kvString("/_source/name", is("Jane Smith")), + kvString("/_source/name", is("John Doe")) + ) + ); + } + + @Test + public void nonCorrelatedNotExistsParentWhere() throws IOException { + String query = String.format(Locale.ROOT, + "SELECT e.name " + + "FROM %s as e " + + "WHERE NOT EXISTS (SELECT * FROM e.projects as p WHERE p.name LIKE 'security') " + + "AND e.name LIKE 'smith'", + TEST_INDEX_EMPLOYEE_NESTED); + + JSONObject response = executeQuery(query); + assertThat( + response, + hitAll( + kvString("/_source/name", is("Susan Smith")) + ) + ); + } + + @Test + public void selectFromSubqueryWithCountShouldPass() throws IOException { + JSONObject result = executeQuery( + StringUtils.format("SELECT t.TEMP as count " + + "FROM (SELECT COUNT(*) as TEMP FROM %s) t", TEST_INDEX_ACCOUNT)); + + assertThat(result.query("/aggregations/count/value"), equalTo(1000)); + } + + @Test + public void selectFromSubqueryWithWhereAndCountShouldPass() throws IOException { + JSONObject result = executeQuery( + StringUtils.format("SELECT t.TEMP as count " + + "FROM (SELECT COUNT(*) as TEMP FROM %s WHERE age > 30) t", TEST_INDEX_ACCOUNT)); + + assertThat(result.query("/aggregations/count/value"), equalTo(502)); + } + + @Test + public void selectFromSubqueryWithCountAndGroupByShouldPass() throws Exception { + JSONObject result = executeQuery( + StringUtils.format("SELECT t.TEMP as count " + + "FROM (SELECT COUNT(*) as TEMP FROM %s GROUP BY gender) t", TEST_INDEX_ACCOUNT)); + + assertThat(getTotalHits(result), equalTo(1000)); + JSONObject gender = (JSONObject) result.query("/aggregations/gender"); + assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); + + boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); + int maleBucketId = isMaleFirst ? 0 : 1; + int femaleBucketId = isMaleFirst ? 1 : 0; + + String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); + String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); + + assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); + assertThat(gender.query(maleBucketPrefix + "/count/value"), equalTo(507)); + assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); + assertThat(gender.query(femaleBucketPrefix + "/count/value"), equalTo(493)); + } + + @Test + public void selectFromSubqueryWithCountAndGroupByAndOrderByShouldPass() throws IOException { + JSONObject result = executeQuery( + StringUtils.format( + "SELECT t.TEMP as count " + + "FROM (SELECT COUNT(*) as TEMP FROM %s GROUP BY age ORDER BY TEMP) t", + TEST_INDEX_ACCOUNT)); + JSONArray buckets = (JSONArray) result.query("/aggregations/age/buckets"); + List countList = new ArrayList<>(); + for (int i = 0; i < buckets.length(); ++i) { + countList.add((int) buckets.query(String.format(Locale.ROOT, "/%d/count/value", i))); + } + + assertTrue(Ordering.natural().isOrdered(countList)); + } + + @Test + public void selectFromSubqueryWithCountAndGroupByAndHavingShouldPass() throws Exception { + JSONObject result = executeQuery( + StringUtils.format("SELECT t.T1 as g, t.T2 as c " + + "FROM (SELECT gender as T1, COUNT(*) as T2 " + + " FROM %s " + + " GROUP BY gender " + + " HAVING T2 > 500) t", TEST_INDEX_ACCOUNT)); + assertThat(result.query("/aggregations/g/buckets/0/c/value"), equalTo(507)); + } + + @Test + public void selectFromSubqueryCountAndSum() throws IOException { + JSONObject result = executeQuery( + StringUtils.format( + "SELECT t.TEMP1 as count, t.TEMP2 as balance " + + "FROM (SELECT COUNT(*) as TEMP1, SUM(balance) as TEMP2 " + + " FROM %s) t", + TEST_INDEX_ACCOUNT)); + + assertThat(result.query("/aggregations/count/value"), equalTo(1000)); + assertThat(result.query("/aggregations/balance/value"), equalTo(25714837.0)); + } + + @Test + public void selectFromSubqueryWithoutAliasShouldPass() throws IOException { + JSONObject response = executeJdbcRequest( + StringUtils.format( + "SELECT a.firstname AS my_first, a.lastname AS my_last, a.age AS my_age " + + "FROM (SELECT firstname, lastname, age " + + "FROM %s " + + "WHERE age = 40 and account_number = 291) AS a", + TEST_INDEX_ACCOUNT)); + + verifySchema(response, + schema("firstname", "my_first", "text"), + schema("lastname", "my_last", "text"), + schema("age", "my_age", "long")); + verifyDataRows(response, + rows("Lynn", "Pollard", 40)); + } + + private JSONObject executeJdbcRequest(String query) { + return new JSONObject(executeQuery(query, "jdbc")); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TermQueryExplainIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TermQueryExplainIT.java new file mode 100644 index 0000000000..f74c11d08c --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TermQueryExplainIT.java @@ -0,0 +1,430 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; + +import java.io.IOException; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.rest.RestStatus; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + + +public class TermQueryExplainIT extends SQLIntegTestCase { + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + loadIndex(Index.ONLINE); + loadIndex(Index.BANK); + loadIndex(Index.BANK_TWO); + loadIndex(Index.DOG); + loadIndex(Index.DOGS2); + loadIndex(Index.DOGS3); + loadIndex(Index.EMPLOYEE_NESTED); + } + + @Test + public void testNonExistingIndex() throws IOException { + try { + explainQuery("SELECT firstname, lastname " + + "FROM elasticsearch_sql_test_fake_index " + + "WHERE firstname = 'Leo'"); + Assert.fail("Expected ResponseException, but none was thrown"); + + } catch (ResponseException e) { + assertThat(e.getResponse().getStatusLine().getStatusCode(), + equalTo(RestStatus.BAD_REQUEST.getStatus())); + final String entity = TestUtils.getResponseBody(e.getResponse()); + assertThat(entity, containsString("no such index")); + assertThat(entity, containsString("\"type\": \"IndexNotFoundException\"")); + } + } + + @Test + public void testNonResolvingIndexPattern() throws IOException { + try { + explainQuery("SELECT * " + + "FROM elasticsearch_sql_test_blah_blah* " + + "WHERE firstname = 'Leo'"); + Assert.fail("Expected ResponseException, but none was thrown"); + + } catch (ResponseException e) { + assertThat(e.getResponse().getStatusLine().getStatusCode(), + equalTo(RestStatus.BAD_REQUEST.getStatus())); + final String entity = TestUtils.getResponseBody(e.getResponse()); + assertThat(entity, containsString("Field [firstname] cannot be found or used here.")); + assertThat(entity, containsString("\"type\": \"SemanticAnalysisException\"")); + } + } + + @Test + public void testNonResolvingIndexPatternWithExistingIndex() throws IOException { + String result = explainQuery( + "SELECT * " + + "FROM elasticsearch_sql_test_blah_blah*, elasticsearch-sql_test_index_bank " + + "WHERE state = 'DC'"); + assertThat(result, containsString("\"term\":{\"state.keyword\"")); + } + + @Test + public void testNonResolvingIndexPatternWithNonExistingIndex() throws IOException { + try { + explainQuery( + "SELECT firstname, lastname " + + "FROM elasticsearch_sql_test_blah_blah*, another_fake_index " + + "WHERE firstname = 'Leo'"); + Assert.fail("Expected ResponseException, but none was thrown"); + } catch (ResponseException e) { + assertThat(e.getResponse().getStatusLine().getStatusCode(), + equalTo(RestStatus.BAD_REQUEST.getStatus())); + final String entity = TestUtils.getResponseBody(e.getResponse()); + assertThat(entity, containsString("no such index")); + assertThat(entity, containsString("\"type\": \"IndexNotFoundException\"")); + } + } + + @Test + public void testNonCompatibleMappings() throws IOException { + try { + explainQuery( + "SELECT * FROM elasticsearch-sql_test_index_dog, elasticsearch-sql_test_index_dog2"); + Assert.fail("Expected ResponseException, but none was thrown"); + } catch (ResponseException e) { + assertThat(e.getResponse().getStatusLine().getStatusCode(), + equalTo(RestStatus.BAD_REQUEST.getStatus())); + final String entity = TestUtils.getResponseBody(e.getResponse()); + assertThat(entity, containsString("Field [holdersName] have conflict type")); + assertThat(entity, containsString("\"type\": \"SemanticAnalysisException\"")); + } + } + + /** + * The dog_name field has same type in dog and dog2 index. + * But, the holdersName field has different type. + */ + @Test + public void testNonCompatibleMappingsButTheFieldIsNotUsed() throws IOException { + String result = explainQuery( + "SELECT dog_name " + + "FROM elasticsearch-sql_test_index_dog, elasticsearch-sql_test_index_dog2 WHERE dog_name = 'dog'"); + System.out.println(result); + assertThat(result, containsString("dog_name")); + assertThat(result, containsString("_source")); + } + + @Test + public void testEqualFieldMappings() throws IOException { + String result = explainQuery( + "SELECT color " + + "FROM elasticsearch-sql_test_index_dog2, elasticsearch-sql_test_index_dog3"); + assertThat(result, containsString("color")); + assertThat(result, containsString("_source")); + } + + @Test + public void testIdenticalMappings() throws IOException { + String result = explainQuery( + "SELECT firstname, birthdate, state " + + "FROM elasticsearch-sql_test_index_bank, elasticsearch-sql_test_index_bank_two " + + "WHERE state = 'WA' OR male = true" + ); + assertThat(result, containsString("term")); + assertThat(result, containsString("state.keyword")); + assertThat(result, containsString("_source")); + } + + @Test + public void testIdenticalMappingsWithTypes() throws IOException { + String result = explainQuery( + "SELECT firstname, birthdate, state " + + "FROM elasticsearch-sql_test_index_bank/account, elasticsearch-sql_test_index_bank_two/account_two " + + "WHERE state = 'WA' OR male = true" + ); + assertThat(result, containsString("term")); + assertThat(result, containsString("state.keyword")); + assertThat(result, containsString("_source")); + } + + + @Test + public void testIdenticalMappingsWithPartialType() throws IOException { + String result = explainQuery( + "SELECT firstname, birthdate, state " + + "FROM elasticsearch-sql_test_index_bank/account, elasticsearch-sql_test_index_bank_two " + + "WHERE state = 'WA' OR male = true" + ); + assertThat(result, containsString("term")); + assertThat(result, containsString("state.keyword")); + assertThat(result, containsString("_source")); + } + + @Test + public void testTextFieldOnly() throws IOException { + + String result = explainQuery( + "SELECT firstname, birthdate, state " + + "FROM elasticsearch-sql_test_index_bank " + + "WHERE firstname = 'Abbas'" + ); + assertThat(result, containsString("term")); + assertThat(result, not(containsString("firstname."))); + } + + @Test + public void testTextAndKeywordAppendsKeywordAlias() throws IOException { + String result = explainQuery( + "SELECT firstname, birthdate, state " + + "FROM elasticsearch-sql_test_index_bank " + + "WHERE state = 'WA' OR lastname = 'Chen'" + ); + assertThat(result, containsString("term")); + assertThat(result, containsString("state.keyword")); + assertThat(result, not(containsString("lastname."))); + } + + @Test + public void testBooleanFieldNoKeywordAlias() throws IOException { + + String result = + explainQuery("SELECT * FROM elasticsearch-sql_test_index_bank WHERE male = false"); + assertThat(result, containsString("term")); + assertThat(result, not(containsString("male."))); + } + + @Test + public void testDateFieldNoKeywordAlias() throws IOException { + + String result = explainQuery( + "SELECT * FROM elasticsearch-sql_test_index_bank WHERE birthdate = '2018-08-19'"); + assertThat(result, containsString("term")); + assertThat(result, not(containsString("birthdate."))); + } + + @Test + public void testNumberNoKeywordAlias() throws IOException { + String result = explainQuery("SELECT * FROM elasticsearch-sql_test_index_bank WHERE age = 32"); + assertThat(result, containsString("term")); + assertThat(result, not(containsString("age."))); + } + + @Test + public void inTestInWhere() throws IOException { + String result = explainQuery( + "SELECT * " + + "FROM elasticsearch-sql_test_index_bank " + + "WHERE state IN ('WA' , 'PA' , 'TN')" + ); + assertThat(result, containsString("term")); + assertThat(result, containsString("state.keyword")); + } + + @Test + @Ignore // TODO: enable when subqueries are fixed + public void inTestInWhereSubquery() throws IOException { + String result = explainQuery( + "SELECT * " + + "FROM elasticsearch-sql_test_index_bank/account WHERE " + + "state IN (SELECT state FROM elasticsearch-sql_test_index_bank WHERE city = 'Nicholson')" + ); + assertThat(result, containsString("term")); + assertThat(result, containsString("state.keyword")); + } + + @Test + public void testKeywordAliasGroupBy() throws IOException { + String result = explainQuery( + "SELECT firstname, state " + + "FROM elasticsearch-sql_test_index_bank/account " + + "GROUP BY firstname, state"); + assertThat(result, containsString("term")); + assertThat(result, containsString("state.keyword")); + } + + @Test + public void testKeywordAliasGroupByUsingTableAlias() throws IOException { + String result = explainQuery( + "SELECT a.firstname, a.state " + + "FROM elasticsearch-sql_test_index_bank/account a " + + "GROUP BY a.firstname, a.state"); + assertThat(result, containsString("term")); + assertThat(result, containsString("state.keyword")); + } + + @Test + public void testKeywordAliasOrderBy() throws IOException { + String result = explainQuery( + "SELECT * " + + "FROM elasticsearch-sql_test_index_bank " + + "ORDER BY state, lastname " + ); + assertThat(result, containsString("\"state.keyword\":{\"order\":\"asc\"")); + assertThat(result, containsString("\"lastname\":{\"order\":\"asc\"}")); + } + + @Test + public void testKeywordAliasOrderByUsingTableAlias() throws IOException { + String result = explainQuery( + "SELECT * " + + "FROM elasticsearch-sql_test_index_bank b " + + "ORDER BY b.state, b.lastname " + ); + assertThat(result, containsString("\"state.keyword\":{\"order\":\"asc\"")); + assertThat(result, containsString("\"lastname\":{\"order\":\"asc\"}")); + } + + @Test + @Ignore // TODO: verify the returned query is correct and fix the expected output + public void testJoinWhere() throws IOException { + String expectedOutput = + TestUtils.fileToString("src/test/resources/expectedOutput/term_join_where", true); + String result = explainQuery( + "SELECT a.firstname, a.lastname , b.city " + + "FROM elasticsearch-sql_test_index_account a " + + "JOIN elasticsearch-sql_test_index_account b " + + "ON a.city = b.city " + + "WHERE a.city IN ('Nicholson', 'Yardville')" + ); + + assertThat(result.replaceAll("\\s+", ""), equalTo(expectedOutput.replaceAll("\\s+", ""))); + } + + @Test + public void testJoinAliasMissing() throws IOException { + try { + explainQuery( + "SELECT a.firstname, a.lastname , b.city " + + "FROM elasticsearch-sql_test_index_account a " + + "JOIN elasticsearch-sql_test_index_account b " + + "ON a.city = b.city " + + "WHERE city IN ('Nicholson', 'Yardville')" + ); + Assert.fail("Expected ResponseException, but none was thrown"); + } catch (ResponseException e) { + assertThat(e.getResponse().getStatusLine().getStatusCode(), + equalTo(RestStatus.BAD_REQUEST.getStatus())); + final String entity = TestUtils.getResponseBody(e.getResponse()); + assertThat(entity, containsString("Field name [city] is ambiguous")); + assertThat(entity, containsString("\"type\": \"VerificationException\"")); + } + + } + + @Test + public void testNestedSingleConditionAllFields() throws IOException { + String result = explainQuery( + "SELECT * " + + "FROM elasticsearch-sql_test_index_employee_nested e, e.projects p " + + "WHERE p.name = 'something' " + ); + assertThat(result, + containsString("\"term\":{\"projects.name.keyword\":{\"value\":\"something\"")); + assertThat(result, containsString("\"path\":\"projects\"")); + } + + @Test + public void testNestedMultipleCondition() throws IOException { + String result = explainQuery( + "SELECT e.id, p.name " + + "FROM elasticsearch-sql_test_index_employee_nested e, e.projects p " + + "WHERE p.name = 'something' and p.started_year = 1990 " + ); + assertThat(result, + containsString("\"term\":{\"projects.name.keyword\":{\"value\":\"something\"")); + assertThat(result, containsString("\"term\":{\"projects.started_year\":{\"value\":1990")); + assertThat(result, containsString("\"path\":\"projects\"")); + } + + @Test + public void testConditionsOnDifferentNestedDocs() throws IOException { + String result = explainQuery( + "SELECT p.name, c.likes " + + "FROM elasticsearch-sql_test_index_employee_nested e, e.projects p, e.comments c " + + "WHERE p.name = 'something' or c.likes = 56 " + ); + assertThat(result, + containsString("\"term\":{\"projects.name.keyword\":{\"value\":\"something\"")); + assertThat(result, containsString("\"term\":{\"comments.likes\":{\"value\":56")); + assertThat(result, containsString("\"path\":\"projects\"")); + assertThat(result, containsString("\"path\":\"comments\"")); + } + + @Test + public void testNestedSingleConditionSpecificFields() throws IOException { + String result = explainQuery( + "SELECT e.id, p.name " + + "FROM elasticsearch-sql_test_index_employee_nested e, e.projects p " + + "WHERE p.name = 'hello' or p.name = 'world' " + ); + assertThat(result, containsString("\"term\":{\"projects.name.keyword\":{\"value\":\"hello\"")); + assertThat(result, containsString("\"term\":{\"projects.name.keyword\":{\"value\":\"world\"")); + assertThat(result, containsString("\"path\":\"projects\"")); + } + + @Test + public void testNestedSingleGroupBy() throws IOException { + String result = explainQuery( + "SELECT e.id, p.name " + + "FROM elasticsearch-sql_test_index_employee_nested e, e.projects p " + + "GROUP BY p.name "); + assertThat(result, containsString("\"terms\":{\"field\":\"projects.name.keyword\"")); + assertThat(result, containsString("\"nested\":{\"path\":\"projects\"")); + } + + @Test + public void testNestedSingleOrderBy() throws IOException { + String result = explainQuery( + "SELECT e.id, p.name " + + "FROM elasticsearch-sql_test_index_employee_nested e, e.projects p " + + "ORDER BY p.name " + ); + assertThat(result, containsString("\"sort\":[{\"projects.name.keyword\"")); + assertThat(result, containsString("\"nested\":{\"path\":\"projects\"")); + } + + @Test + public void testNestedIsNotNullExplain() throws IOException { + String explain = explainQuery( + "SELECT e.name " + + "FROM elasticsearch-sql_test_index_employee_nested as e, e.projects as p " + + "WHERE p IS NOT NULL" + ); + + assertThat(explain, containsString("\"exists\":{\"field\":\"projects\"")); + assertThat(explain, containsString("\"path\":\"projects\"")); + } + + @Test + @Ignore // TODO: enable when subqueries are fixed + public void testMultiQuery() throws IOException { + String expectedOutput = + TestUtils.fileToString("src/test/resources/expectedOutput/term_union_where", true); + String result = explainQuery( + "SELECT firstname " + + "FROM elasticsearch-sql_test_index_account/account " + + "WHERE firstname = 'Amber' " + + "UNION ALL " + + "SELECT dog_name as firstname " + + "FROM elasticsearch-sql_test_index_dog/dog " + + "WHERE holdersName = 'Hattie' OR dog_name = 'rex'"); + assertThat(result.replaceAll("\\s+", ""), equalTo(expectedOutput.replaceAll("\\s+", ""))); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestUtils.java new file mode 100644 index 0000000000..a44715fb89 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestUtils.java @@ -0,0 +1,396 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static com.google.common.base.Strings.isNullOrEmpty; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.xcontent.XContentType; +import org.json.JSONObject; + +public class TestUtils { + + private final static String MAPPING_FILE_PATH = "src/test/resources/indexDefinitions/"; + + /** + * Create test index by REST client. + * + * @param client client connection + * @param indexName test index name + * @param mapping test index mapping or null if no predefined mapping + */ + public static void createIndexByRestClient(RestClient client, String indexName, String mapping) { + Request request = new Request("PUT", "/" + indexName); + if (!isNullOrEmpty(mapping)) { + request.setJsonEntity(mapping); + } + performRequest(client, request); + } + + /** + * https://github.com/elastic/elasticsearch/pull/49959 + * Deprecate creation of dot-prefixed index names except for hidden and system indices. + * Create hidden index by REST client. + * + * @param client client connection + * @param indexName test index name + * @param mapping test index mapping or null if no predefined mapping + */ + public static void createHiddenIndexByRestClient(RestClient client, String indexName, + String mapping) { + Request request = new Request("PUT", "/" + indexName); + JSONObject jsonObject = isNullOrEmpty(mapping) ? new JSONObject() : new JSONObject(mapping); + jsonObject.put("settings", new JSONObject("{\"index\":{\"hidden\":true}}")); + request.setJsonEntity(jsonObject.toString()); + + performRequest(client, request); + } + + /** + * Check if index already exists by ES index exists API which returns: + * 200 - specified indices or aliases exist + * 404 - one or more indices specified or aliases do not exist + * + * @param client client connection + * @param indexName index name + * @return true for index exist + */ + public static boolean isIndexExist(RestClient client, String indexName) { + try { + Response response = client.performRequest(new Request("HEAD", "/" + indexName)); + return (response.getStatusLine().getStatusCode() == 200); + } catch (IOException e) { + throw new IllegalStateException("Failed to perform request", e); + } + } + + /** + * Load test data set by REST client. + * + * @param client client connection + * @param indexName index name + * @param dataSetFilePath file path of test data set + * @throws IOException + */ + public static void loadDataByRestClient(RestClient client, String indexName, + String dataSetFilePath) throws IOException { + Path path = Paths.get(getResourceFilePath(dataSetFilePath)); + Request request = new Request("POST", "/" + indexName + "/_bulk?refresh=true"); + request.setJsonEntity(new String(Files.readAllBytes(path))); + performRequest(client, request); + } + + /** + * Perform a request by REST client. + * + * @param client client connection + * @param request request object + */ + public static Response performRequest(RestClient client, Request request) { + try { + Response response = client.performRequest(request); + int status = response.getStatusLine().getStatusCode(); + if (status >= 400) { + throw new IllegalStateException("Failed to perform request. Error code: " + status); + } + return response; + } catch (IOException e) { + throw new IllegalStateException("Failed to perform request", e); + } + } + + public static String getAccountIndexMapping() { + String mappingFile = "account_index_mapping.json"; + return getMappingFile(mappingFile); + } + + public static String getPhraseIndexMapping() { + String mappingFile = "phrase_index_mapping.json"; + return getMappingFile(mappingFile); + } + + public static String getDogIndexMapping() { + String mappingFile = "dog_index_mapping.json"; + return getMappingFile(mappingFile); + } + + public static String getDogs2IndexMapping() { + String mappingFile = "dog2_index_mapping.json"; + return getMappingFile(mappingFile); + } + + public static String getDogs3IndexMapping() { + String mappingFile = "dog3_index_mapping.json"; + return getMappingFile(mappingFile); + } + + public static String getPeople2IndexMapping() { + String mappingFile = "people2_index_mapping.json"; + return getMappingFile(mappingFile); + } + + public static String getGameOfThronesIndexMapping() { + String mappingFile = "game_of_thrones_index_mapping.json"; + return getMappingFile(mappingFile); + } + + // System + + public static String getOdbcIndexMapping() { + String mappingFile = "odbc_index_mapping.json"; + return getMappingFile(mappingFile); + } + + public static String getLocationIndexMapping() { + String mappingFile = "location_index_mapping.json"; + return getMappingFile(mappingFile); + } + + public static String getEmployeeNestedTypeIndexMapping() { + String mappingFile = "employee_nested_type_index_mapping.json"; + return getMappingFile(mappingFile); + } + + + public static String getNestedTypeIndexMapping() { + String mappingFile = "nested_type_index_mapping.json"; + return getMappingFile(mappingFile); + } + + public static String getJoinTypeIndexMapping() { + String mappingFile = "join_type_index_mapping.json"; + return getMappingFile(mappingFile); + } + + public static String getBankIndexMapping() { + String mappingFile = "bank_index_mapping.json"; + return getMappingFile(mappingFile); + } + + public static String getBankWithNullValuesIndexMapping() { + String mappingFile = "bank_with_null_values_index_mapping.json"; + return getMappingFile(mappingFile); + } + + public static String getOrderIndexMapping() { + String mappingFile = "order_index_mapping.json"; + return getMappingFile(mappingFile); + } + + public static String getWeblogsIndexMapping() { + String mappingFile = "weblogs_index_mapping.json"; + return getMappingFile(mappingFile); + } + + public static String getDateIndexMapping() { + String mappingFile = "date_index_mapping.json"; + return getMappingFile(mappingFile); + } + + public static String getDateTimeIndexMapping() { + String mappingFile = "date_time_index_mapping.json"; + return getMappingFile(mappingFile); + } + + public static String getNestedSimpleIndexMapping() { + String mappingFile = "nested_simple_index_mapping.json"; + return getMappingFile(mappingFile); + } + + public static void loadBulk(Client client, String jsonPath, String defaultIndex) + throws Exception { + System.out.println(String.format("Loading file %s into elasticsearch cluster", jsonPath)); + String absJsonPath = getResourceFilePath(jsonPath); + + BulkRequest bulkRequest = new BulkRequest(); + try (final InputStream stream = new FileInputStream(absJsonPath); + final Reader streamReader = new InputStreamReader(stream, StandardCharsets.UTF_8); + final BufferedReader br = new BufferedReader(streamReader)) { + + while (true) { + + String actionLine = br.readLine(); + if (actionLine == null || actionLine.trim().isEmpty()) { + break; + } + String sourceLine = br.readLine(); + JSONObject actionJson = new JSONObject(actionLine); + + IndexRequest indexRequest = new IndexRequest(); + indexRequest.index(defaultIndex); + String docType = actionJson.getJSONObject("index").getString("_type"); + indexRequest.type(docType); + if (actionJson.getJSONObject("index").has("_id")) { + String docId = actionJson.getJSONObject("index").getString("_id"); + indexRequest.id(docId); + } + if (actionJson.getJSONObject("index").has("_routing")) { + String routing = actionJson.getJSONObject("index").getString("_routing"); + indexRequest.routing(routing); + } + indexRequest.source(sourceLine, XContentType.JSON); + bulkRequest.add(indexRequest); + } + } + + BulkResponse bulkResponse = client.bulk(bulkRequest).actionGet(); + + if (bulkResponse.hasFailures()) { + throw new Exception("Failed to load test data into index " + defaultIndex + ", " + + bulkResponse.buildFailureMessage()); + } + System.out.println(bulkResponse.getItems().length + " documents loaded."); + // ensure the documents are searchable + client.admin().indices().prepareRefresh(defaultIndex).execute().actionGet(); + } + + public static String getResourceFilePath(String relPath) { + String projectRoot = System.getProperty("project.root", null); + if (projectRoot == null) { + return new File(relPath).getAbsolutePath(); + } else { + return new File(projectRoot + "/" + relPath).getAbsolutePath(); + } + } + + public static String getResponseBody(Response response) throws IOException { + + return getResponseBody(response, false); + } + + public static String getResponseBody(Response response, boolean retainNewLines) + throws IOException { + final StringBuilder sb = new StringBuilder(); + + try (final InputStream is = response.getEntity().getContent(); + final BufferedReader br = new BufferedReader( + new InputStreamReader(is, StandardCharsets.UTF_8))) { + + String line; + while ((line = br.readLine()) != null) { + sb.append(line); + if (retainNewLines) { + sb.append(String.format(Locale.ROOT, "%n")); + } + } + } + return sb.toString(); + } + + public static String fileToString(final String filePathFromProjectRoot, + final boolean removeNewLines) + throws IOException { + + final String absolutePath = getResourceFilePath(filePathFromProjectRoot); + + try (final InputStream stream = new FileInputStream(absolutePath); + final Reader streamReader = new InputStreamReader(stream, StandardCharsets.UTF_8); + final BufferedReader br = new BufferedReader(streamReader)) { + + final StringBuilder stringBuilder = new StringBuilder(); + String line = br.readLine(); + + while (line != null) { + + stringBuilder.append(line); + if (!removeNewLines) { + stringBuilder.append(String.format(Locale.ROOT, "%n")); + } + line = br.readLine(); + } + + return stringBuilder.toString(); + } + } + + /** + * Builds all permutations of the given list of Strings + * + * @param items list of strings to permute + * @return list of permutations + */ + public static List> getPermutations(final List items) { + + if (items.size() > 5) { + throw new IllegalArgumentException("Inefficient test, please refactor"); + } + + final List> result = new LinkedList<>(); + + if (items.isEmpty() || 1 == items.size()) { + + final List onlyElement = new ArrayList<>(); + if (1 == items.size()) { + onlyElement.add(items.get(0)); + } + result.add(onlyElement); + return result; + } + + for (int i = 0; i < items.size(); ++i) { + + final List smallerSet = new ArrayList<>(); + + if (i != 0) { + smallerSet.addAll(items.subList(0, i)); + } + if (i != items.size() - 1) { + smallerSet.addAll(items.subList(i + 1, items.size())); + } + + final String currentItem = items.get(i); + result.addAll(getPermutations(smallerSet).stream().map(smallerSetPermutation -> { + final List permutation = new ArrayList<>(); + permutation.add(currentItem); + permutation.addAll(smallerSetPermutation); + return permutation; + }).collect(Collectors.toCollection(LinkedList::new))); + } + + return result; + } + + public static String getMappingFile(String fileName) { + try { + return fileToString(MAPPING_FILE_PATH + fileName, false); + } catch (IOException e) { + return null; + } + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestsConstants.java new file mode 100644 index 0000000000..30f07d73b5 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestsConstants.java @@ -0,0 +1,62 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +/** + * Created by omershelef on 18/12/14. + */ +public class TestsConstants { + + public final static String PERSISTENT = "persistent"; + public final static String TRANSIENT = "transient"; + + public final static String TEST_INDEX = "elasticsearch-sql_test_index"; + + public final static String TEST_INDEX_ONLINE = TEST_INDEX + "_online"; + public final static String TEST_INDEX_ACCOUNT = TEST_INDEX + "_account"; + public final static String TEST_INDEX_PHRASE = TEST_INDEX + "_phrase"; + public final static String TEST_INDEX_DOG = TEST_INDEX + "_dog"; + public final static String TEST_INDEX_DOG2 = TEST_INDEX + "_dog2"; + public final static String TEST_INDEX_DOG3 = TEST_INDEX + "_dog3"; + public final static String TEST_INDEX_DOGSUBQUERY = TEST_INDEX + "_subquery"; + public final static String TEST_INDEX_PEOPLE = TEST_INDEX + "_people"; + public final static String TEST_INDEX_PEOPLE2 = TEST_INDEX + "_people2"; + public final static String TEST_INDEX_GAME_OF_THRONES = TEST_INDEX + "_game_of_thrones"; + public final static String TEST_INDEX_SYSTEM = TEST_INDEX + "_system"; + public final static String TEST_INDEX_ODBC = TEST_INDEX + "_odbc"; + public final static String TEST_INDEX_LOCATION = TEST_INDEX + "_location"; + public final static String TEST_INDEX_LOCATION2 = TEST_INDEX + "_location2"; + public final static String TEST_INDEX_NESTED_TYPE = TEST_INDEX + "_nested_type"; + public final static String TEST_INDEX_NESTED_SIMPLE = TEST_INDEX + "_nested_simple"; + public final static String TEST_INDEX_NESTED_WITH_QUOTES = + TEST_INDEX + "_nested_type_with_quotes"; + public final static String TEST_INDEX_EMPLOYEE_NESTED = TEST_INDEX + "_employee_nested"; + public final static String TEST_INDEX_JOIN_TYPE = TEST_INDEX + "_join_type"; + public final static String TEST_INDEX_BANK = TEST_INDEX + "_bank"; + public final static String TEST_INDEX_BANK_TWO = TEST_INDEX_BANK + "_two"; + public final static String TEST_INDEX_BANK_WITH_NULL_VALUES = + TEST_INDEX_BANK + "_with_null_values"; + public final static String TEST_INDEX_ORDER = TEST_INDEX + "_order"; + public final static String TEST_INDEX_WEBLOG = TEST_INDEX + "_weblog"; + public final static String TEST_INDEX_DATE = TEST_INDEX + "_date"; + public final static String TEST_INDEX_DATE_TIME = TEST_INDEX + "_datetime"; + + + public final static String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; + public final static String TS_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; + public final static String SIMPLE_DATE_FORMAT = "yyyy-MM-dd"; +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TypeInformationIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TypeInformationIT.java new file mode 100644 index 0000000000..d674894b43 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TypeInformationIT.java @@ -0,0 +1,166 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy; + +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.schema; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifySchema; + +import org.json.JSONObject; +import org.junit.Test; + +public class TypeInformationIT extends SQLIntegTestCase { + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + loadIndex(Index.ONLINE); + } + + /* + numberOperators + */ + @Test + public void testAbsWithIntFieldReturnsInt() { + JSONObject response = + executeJdbcRequest("SELECT ABS(age) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + + " ORDER BY age LIMIT 5"); + + verifySchema(response, schema("ABS(age)", null, "long")); + } + + @Test + public void testCeilWithLongFieldReturnsLong() { + JSONObject response = + executeJdbcRequest("SELECT CEIL(balance) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + + " ORDER BY balance LIMIT 5"); + + verifySchema(response, schema("CEIL(balance)", null, "long")); + } + + /* + mathConstants + */ + @Test + public void testPiReturnsDouble() { + JSONObject response = executeJdbcRequest("SELECT PI() FROM " + TestsConstants.TEST_INDEX_ACCOUNT + + " LIMIT 1"); + + verifySchema(response, schema("PI()", null, "double")); + } + + /* + stringOperators + */ + @Test + public void testUpperWithStringFieldReturnsString() { + JSONObject response = executeJdbcRequest("SELECT UPPER(firstname) AS firstname_alias FROM " + + TestsConstants.TEST_INDEX_ACCOUNT + " ORDER BY firstname_alias LIMIT 2"); + + verifySchema(response, schema("firstname_alias", null, "text")); + } + + @Test + public void testLowerWithTextFieldReturnsText() { + JSONObject response = executeJdbcRequest("SELECT LOWER(firstname) FROM " + + TestsConstants.TEST_INDEX_ACCOUNT + " ORDER BY firstname LIMIT 2"); + + verifySchema(response, schema("LOWER(firstname)", null, "text")); + } + + /* + stringFunctions + */ + @Test + public void testLengthWithTextFieldReturnsInt() { + JSONObject response = executeJdbcRequest("SELECT length(firstname) FROM " + + TestsConstants.TEST_INDEX_ACCOUNT + " ORDER BY firstname LIMIT 2"); + + verifySchema(response, schema("length(firstname)", null, "integer")); + } + + @Test + public void testLengthWithGroupByExpr() { + JSONObject response = + executeJdbcRequest("SELECT Length(firstname) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + + " GROUP BY LENGTH(firstname) LIMIT 5"); + + verifySchema(response, schema("Length(firstname)", null, "integer")); + } + + /* + trigFunctions + */ + @Test + public void testSinWithLongFieldReturnsDouble() { + JSONObject response = executeJdbcRequest("SELECT sin(balance) FROM " + + TestsConstants.TEST_INDEX_ACCOUNT + " ORDER BY firstname LIMIT 2"); + + verifySchema(response, schema("sin(balance)", null, "double")); + } + + @Test + public void testRadiansWithLongFieldReturnsDouble() { + JSONObject response = executeJdbcRequest("SELECT radians(balance) FROM " + + TestsConstants.TEST_INDEX_ACCOUNT + " ORDER BY firstname LIMIT 2"); + + verifySchema(response, schema("radians(balance)", null, "double")); + } + + /* + binaryOperators + */ + @Test + public void testAddWithIntReturnsInt() { + JSONObject response = executeJdbcRequest("SELECT (balance + 5) AS balance_add_five FROM " + + TestsConstants.TEST_INDEX_ACCOUNT + " ORDER BY firstname LIMIT 2"); + + verifySchema(response, schema("balance_add_five", null, "integer")); + } + + @Test + public void testSubtractLongWithLongReturnsLong() { + JSONObject response = executeJdbcRequest("SELECT (balance - balance) FROM " + + TestsConstants.TEST_INDEX_ACCOUNT + " ORDER BY firstname LIMIT 2"); + + verifySchema(response, schema("subtract(balance, balance)", null, "long")); + } + + /* + dateFunctions + */ + @Test + public void testDayOfWeekWithKeywordReturnsText() { + JSONObject response = executeJdbcRequest("SELECT DAY_OF_WEEK(insert_time) FROM " + + TestsConstants.TEST_INDEX_ONLINE + " LIMIT 2"); + + verifySchema(response, + schema("DAY_OF_WEEK(insert_time)", null, "integer")); + } + + @Test + public void testYearWithKeywordReturnsText() { + JSONObject response = executeJdbcRequest("SELECT YEAR(insert_time) FROM " + + TestsConstants.TEST_INDEX_ONLINE + " LIMIT 2"); + + verifySchema(response, schema("YEAR(insert_time)", null, "integer")); + } + + private JSONObject executeJdbcRequest(String query) { + return new JSONObject(executeQuery(query, "jdbc")); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/DedupCommandIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/DedupCommandIT.java new file mode 100644 index 0000000000..39e6d128e7 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/DedupCommandIT.java @@ -0,0 +1,76 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl; + +import org.json.JSONObject; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.rows; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyDataRows; + +public class DedupCommandIT extends PPLIntegTestCase { + + @Override + public void init() throws IOException { + loadIndex(Index.BANK); + loadIndex(Index.BANK_WITH_NULL_VALUES); + } + + @Test + public void testDedup() throws IOException { + JSONObject result = + executeQuery(String.format("source=%s | dedup male | fields male", TEST_INDEX_BANK)); + verifyDataRows(result, rows(true), rows(false)); + } + + @Test + public void testConsecutiveDedup() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | dedup male consecutive=true | fields male", TEST_INDEX_BANK)); + verifyDataRows(result, rows(true), rows(false), rows(true), rows(false)); + } + + @Test + public void testAllowMoreDuplicates() throws IOException { + JSONObject result = + executeQuery(String.format("source=%s | dedup 2 male | fields male", TEST_INDEX_BANK)); + verifyDataRows(result, rows(true), rows(true), rows(false), rows(false)); + } + + @Test + public void testKeepEmptyDedup() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | dedup balance keepempty=true | fields firstname, balance", + TEST_INDEX_BANK_WITH_NULL_VALUES)); + verifyDataRows( + result, + rows("Amber JOHnny", 39225), + rows("Hattie"), + rows("Nanette", 32838), + rows("Dale", 4180), + rows("Elinor"), + rows("Virginia"), + rows("Dillard", 48086)); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/FieldsCommandIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/FieldsCommandIT.java new file mode 100644 index 0000000000..e4c5a0f4bd --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/FieldsCommandIT.java @@ -0,0 +1,57 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl; + +import org.json.JSONObject; +import org.junit.Ignore; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.columnName; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.columnPattern; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyColumn; + +public class FieldsCommandIT extends PPLIntegTestCase { + + @Override + public void init() throws IOException { + loadIndex(Index.ACCOUNT); + } + + @Test + public void testFieldsWithOneField() throws IOException { + JSONObject result = + executeQuery(String.format("source=%s | fields firstname", TEST_INDEX_ACCOUNT)); + verifyColumn(result, columnName("firstname")); + } + + @Test + public void testFieldsWithMultiFields() throws IOException { + JSONObject result = + executeQuery(String.format("source=%s | fields firstname, lastname", TEST_INDEX_ACCOUNT)); + verifyColumn(result, columnName("firstname"), columnName("lastname")); + } + + @Ignore("Cannot resolve wildcard yet") + @Test + public void testFieldsWildCard() throws IOException { + JSONObject result = + executeQuery(String.format("source=%s | fields ", TEST_INDEX_ACCOUNT) + "firstnam%"); + verifyColumn(result, columnPattern("^firstnam.*")); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/OperatorIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/OperatorIT.java new file mode 100644 index 0000000000..5e632ce3a0 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/OperatorIT.java @@ -0,0 +1,307 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.rows; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyDataRows; + +import java.io.IOException; +import org.elasticsearch.client.ResponseException; +import org.hamcrest.Matchers; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; + +public class OperatorIT extends PPLIntegTestCase { + @Override + public void init() throws IOException { + loadIndex(Index.BANK); + loadIndex(Index.BANK_WITH_NULL_VALUES); + } + + @Test + public void testAddOperator() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | where age = 31 + 1 | fields age", + TEST_INDEX_BANK)); + verifyDataRows(result, rows(32)); + } + + @Test + public void testSubtractOperator() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | where age = 33 - 1 | fields age", + TEST_INDEX_BANK)); + verifyDataRows(result, rows(32)); + } + + @Test + public void testMultiplyOperator() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | where age = 16 * 2 | fields age", + TEST_INDEX_BANK)); + verifyDataRows(result, rows(32)); + } + + @Test + public void testDivideOperator() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | where age / 2 = 16 | fields age", + TEST_INDEX_BANK)); + verifyDataRows(result, rows(32), rows(33)); + } + + @Test + public void testModuleOperator() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | where age %s 32 = 0 | fields age", + TEST_INDEX_BANK, "%")); + verifyDataRows(result, rows(32)); + } + + @Test + public void testArithmeticOperatorWithNullValue() throws IOException { + String result = + executeQueryToString( + String.format( + "source=%s | eval f = age + 0 | fields f", + TEST_INDEX_BANK_WITH_NULL_VALUES)); + assertEquals( + "{\n" + + " \"schema\": [{\n" + + " \"name\": \"f\",\n" + + " \"type\": \"integer\"\n" + + " }],\n" + + " \"total\": 7,\n" + + " \"datarows\": [\n" + + " [32],\n" + + " [36],\n" + + " [28],\n" + + " [33],\n" + + " [36],\n" + + " [null],\n" + + " [34]\n" + + " ],\n" + + " \"size\": 7\n" + + "}\n", + result); + } + + @Test + public void testArithmeticOperatorWithMissingValue() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | eval f = balance * 1 | fields f", TEST_INDEX_BANK_WITH_NULL_VALUES)); + verifyDataRows( + result, rows(39225), rows(32838), rows(4180), rows(48086), rows(), rows(), rows()); + } + + @Test + public void testMultipleArithmeticOperators() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | where (age+2) * 3 / 2 - 1 = 50 | fields age", + TEST_INDEX_BANK)); + verifyDataRows(result, rows(32)); + } + + @Test + public void testAndOperator() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | where firstname='Amber JOHnny' and age=32 | fields firstname, age", + TEST_INDEX_BANK)); + verifyDataRows(result, rows("Amber JOHnny", 32)); + + result = + executeQuery( + String.format( + "source=%s | where age=32 and firstname='Amber JOHnny' | fields firstname, age", + TEST_INDEX_BANK)); + verifyDataRows(result, rows("Amber JOHnny", 32)); + } + + @Test + public void testOrOperator() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | where age=32 or age=34 | fields age", TEST_INDEX_BANK)); + verifyDataRows(result, rows(32), rows(34)); + + result = + executeQuery( + String.format( + "source=%s | where age=34 or age=32| fields age", TEST_INDEX_BANK)); + verifyDataRows(result, rows(32), rows(34)); + } + + @Test + public void testXorOperator() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | where firstname='Hattie' xor age=36 | fields firstname, age", + TEST_INDEX_BANK)); + verifyDataRows(result, rows("Elinor", 36)); + + result = + executeQuery( + String.format( + "source=%s | where age=36 xor firstname='Hattie' | fields firstname, age", + TEST_INDEX_BANK)); + verifyDataRows(result, rows("Elinor", 36)); + } + + @Test + public void testNotOperator() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s not age > 32 | fields age", + TEST_INDEX_BANK)); + verifyDataRows(result, rows(28), rows(32)); + } + + @Test + public void testEqualOperator() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s age = 32 | fields age", + TEST_INDEX_BANK)); + verifyDataRows(result, rows(32)); + + result = + executeQuery( + String.format( + "source=%s 32 = age | fields age", + TEST_INDEX_BANK)); + verifyDataRows(result, rows(32)); + } + + @Test + public void testNotEqualOperator() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s age != 32 | fields age", + TEST_INDEX_BANK)); + verifyDataRows(result, rows(28), rows(33), rows(34), rows(36), rows(36), rows(39)); + + result = + executeQuery( + String.format( + "source=%s 32 != age | fields age", + TEST_INDEX_BANK)); + verifyDataRows(result, rows(28), rows(33), rows(34), rows(36), rows(36), rows(39)); + } + + @Test + public void testLessOperator() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s age < 32 | fields age", + TEST_INDEX_BANK)); + verifyDataRows(result, rows(28)); + } + + @Test + public void testLteOperator() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s age <= 32 | fields age", + TEST_INDEX_BANK)); + verifyDataRows(result, rows(28), rows(32)); + } + + @Test + public void testGreaterOperator() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s age > 36 | fields age", + TEST_INDEX_BANK)); + verifyDataRows(result, rows(39)); + } + + @Test + public void testGteOperator() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s age >= 36 | fields age", + TEST_INDEX_BANK)); + verifyDataRows(result, rows(36), rows(36), rows(39)); + } + + @Test + public void testLikeOperator() throws IOException { + JSONObject result = + executeQuery( + String.format("source=%s firstname like 'Hatti_' | fields firstname", TEST_INDEX_BANK)); + verifyDataRows(result, rows("Hattie")); + } + + @Test + public void testBinaryPredicateWithNullValue() { + queryExecutionShouldThrowExceptionDueToNullOrMissingValue( + String.format("source=%s | where age < 32", TEST_INDEX_BANK_WITH_NULL_VALUES), + "invalid to call type operation on null value" + ); + } + + @Test + public void testBinaryPredicateWithMissingValue() { + queryExecutionShouldThrowExceptionDueToNullOrMissingValue( + String.format("source=%s | where balance > 3000", TEST_INDEX_BANK_WITH_NULL_VALUES), + "invalid to call type operation on missing value" + ); + } + + private void queryExecutionShouldThrowExceptionDueToNullOrMissingValue( + String query, String... errorMsgs) { + try { + executeQuery(query); + fail("Expected to throw ExpressionEvaluationException, but none was thrown for query: " + + query); + } catch (ResponseException e) { + String errorMsg = e.getMessage(); + assertTrue(errorMsg.contains("ExpressionEvaluationException")); + for (String msg: errorMsgs) { + assertTrue(errorMsg.contains(msg)); + } + } catch (IOException e) { + throw new IllegalStateException("Unexpected exception raised for query: " + query); + } + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/PPLIntegTestCase.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/PPLIntegTestCase.java new file mode 100644 index 0000000000..87c7cb9bd7 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/PPLIntegTestCase.java @@ -0,0 +1,64 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl; + +import com.amazon.opendistroforelasticsearch.sql.legacy.SQLIntegTestCase; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.json.JSONException; +import org.json.JSONObject; +import org.junit.Assert; + +import java.io.IOException; +import java.util.Locale; + +import static com.amazon.opendistroforelasticsearch.sql.plugin.rest.RestPPLQueryAction.QUERY_API_ENDPOINT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getResponseBody; + +/** + * ES Rest integration test base for PPL testing + */ +public abstract class PPLIntegTestCase extends SQLIntegTestCase { + + protected JSONObject executeQuery(String query) throws IOException { + return jsonify(executeQueryToString(query)); + } + + protected String executeQueryToString(String query) throws IOException { + Response response = client().performRequest(buildRequest(query)); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + return getResponseBody(response, true); + } + + protected Request buildRequest(String query) { + Request request = new Request("POST", QUERY_API_ENDPOINT); + request.setJsonEntity(String.format(Locale.ROOT, "{\n" + " \"query\": \"%s\"\n" + "}", query)); + + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + request.setOptions(restOptionsBuilder); + return request; + } + + private JSONObject jsonify(String text) { + try { + return new JSONObject(text); + } catch (JSONException e) { + throw new IllegalStateException(String.format("Failed to transform %s to JSON format", text)); + } + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/PPLPluginIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/PPLPluginIT.java new file mode 100644 index 0000000000..86219cc5cd --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/PPLPluginIT.java @@ -0,0 +1,86 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl; + +import static org.hamcrest.Matchers.hasProperty; + +import java.io.IOException; +import java.util.Locale; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.hamcrest.Description; +import org.hamcrest.TypeSafeMatcher; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +public class PPLPluginIT extends PPLIntegTestCase { + @Rule public ExpectedException exceptionRule = ExpectedException.none(); + + @Override + protected void init() throws Exception { + wipeAllClusterSettings(); + } + + @Test + public void testQueryEndpointShouldOK() throws IOException { + Request request = new Request("PUT", "/a/_doc/1?refresh=true"); + request.setJsonEntity("{\"name\": \"hello\"}"); + client().performRequest(request); + + String response = executeQueryToString("search source=a"); + assertEquals( + "{\n" + + " \"schema\": [{\n" + + " \"name\": \"name\",\n" + + " \"type\": \"string\"\n" + + " }],\n" + + " \"total\": 1,\n" + + " \"datarows\": [[\"hello\"]],\n" + + " \"size\": 1\n" + + "}\n", + response); + } + + @Test + public void testQueryEndpointShouldFail() throws IOException { + exceptionRule.expect(ResponseException.class); + exceptionRule.expect(hasProperty("response", statusCode(500))); + + client().performRequest(makePPLRequest("search invalid")); + } + + protected Request makePPLRequest(String query) { + Request post = new Request("POST", "/_opendistro/_ppl"); + post.setJsonEntity(String.format(Locale.ROOT, "{\n" + " \"query\": \"%s\"\n" + "}", query)); + return post; + } + + private TypeSafeMatcher statusCode(int statusCode) { + return new TypeSafeMatcher() { + @Override + public void describeTo(Description description) { + description.appendText(String.format(Locale.ROOT, "statusCode=%d", statusCode)); + } + + @Override + protected boolean matchesSafely(Response resp) { + return resp.getStatusLine().getStatusCode() == statusCode; + } + }; + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/QueryAnalysisIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/QueryAnalysisIT.java new file mode 100644 index 0000000000..5a1872de22 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/QueryAnalysisIT.java @@ -0,0 +1,170 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; + +import com.amazon.opendistroforelasticsearch.sql.common.antlr.SyntaxCheckException; +import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; +import java.io.IOException; +import org.elasticsearch.client.ResponseException; +import org.junit.Ignore; +import org.junit.Test; + +public class QueryAnalysisIT extends PPLIntegTestCase { + + @Override + public void init() throws IOException { + loadIndex(Index.ACCOUNT); + } + + /** Valid commands should pass both syntax analysis and semantic check. */ + @Test + public void searchCommandShouldPassSemanticCheck() { + String query = String.format("search source=%s age=20", TEST_INDEX_ACCOUNT); + queryShouldPassSyntaxAndSemanticCheck(query); + } + + @Test + public void whereCommandShouldPassSemanticCheck() { + String query = String.format("search source=%s | where age=20", TEST_INDEX_ACCOUNT); + queryShouldPassSyntaxAndSemanticCheck(query); + } + + @Test + public void fieldsCommandShouldPassSemanticCheck() { + String query = String.format("search source=%s | fields firstname", TEST_INDEX_ACCOUNT); + queryShouldPassSyntaxAndSemanticCheck(query); + } + + @Ignore("Can't resolve target field yet") + @Test + public void renameCommandShouldPassSemanticCheck() { + String query = + String.format("search source=%s | rename firstname as first", TEST_INDEX_ACCOUNT); + queryShouldPassSyntaxAndSemanticCheck(query); + } + + @Test + public void statsCommandShouldPassSemanticCheck() { + String query = String.format("search source=%s | stats avg(age)", TEST_INDEX_ACCOUNT); + queryShouldPassSyntaxAndSemanticCheck(query); + } + + @Test + public void dedupCommandShouldPassSemanticCheck() { + String query = + String.format("search source=%s | dedup firstname, lastname", TEST_INDEX_ACCOUNT); + queryShouldPassSyntaxAndSemanticCheck(query); + } + + @Test + public void sortCommandShouldPassSemanticCheck() { + String query = String.format("search source=%s | sort age", TEST_INDEX_ACCOUNT); + queryShouldPassSyntaxAndSemanticCheck(query); + } + + @Test + public void evalCommandShouldPassSemanticCheck() { + String query = String.format("search source=%s | eval age=abs(age)", TEST_INDEX_ACCOUNT); + queryShouldPassSyntaxAndSemanticCheck(query); + queryShouldPassSyntaxAndSemanticCheck(query); + } + + @Test + public void queryShouldBeCaseInsensitiveInKeywords() { + String query = String.format("SEARCH SourCE=%s", TEST_INDEX_ACCOUNT); + queryShouldPassSyntaxAndSemanticCheck(query); + } + + /** + * Commands that fail syntax analysis should throw + * {@link SyntaxCheckException}. + */ + @Test + public void queryNotStartingWithSearchCommandShouldFailSyntaxCheck() { + String query = "fields firstname"; + queryShouldThrowSyntaxException(query, "Failed to parse query due to offending symbol"); + } + + @Test + public void queryWithIncorrectCommandShouldFailSyntaxCheck() { + String query = String.format("search source=%s | field firstname", TEST_INDEX_ACCOUNT); + queryShouldThrowSyntaxException(query, "Failed to parse query due to offending symbol"); + } + + @Test + public void queryWithIncorrectKeywordsShouldFailSyntaxCheck() { + String query = String.format("search sources=%s", TEST_INDEX_ACCOUNT); + queryShouldThrowSyntaxException(query, "Failed to parse query due to offending symbol"); + } + + @Test + public void unsupportedAggregationFunctionShouldFailSyntaxCheck() { + String query = String.format("search source=%s | stats range(age)", TEST_INDEX_ACCOUNT); + queryShouldThrowSyntaxException(query, "Failed to parse query due to offending symbol"); + } + + /** + * Commands that fail semantic analysis should throw {@link SemanticCheckException}. + */ + @Test + public void nonexistentFieldShouldFailSemanticCheck() { + String query = String.format("search source=%s | fields name", TEST_INDEX_ACCOUNT); + queryShouldThrowSemanticException(query, "can't resolve Symbol(namespace=FIELD_NAME, " + + "name=name) in type env"); + } + + private void queryShouldPassSyntaxAndSemanticCheck(String query) { + try { + executeQuery(query); + } catch (SemanticCheckException e) { + fail("Expected to pass semantic check but failed for query: " + query); + } catch (IOException e) { + throw new IllegalStateException("Unexpected IOException raised for query: " + query); + } + } + + private void queryShouldThrowSyntaxException(String query, String... messages) { + try { + executeQuery(query); + fail("Expected to throw SyntaxCheckException, but none was thrown for query: " + query); + } catch (ResponseException e) { + String errorMsg = e.getMessage(); + assertTrue(errorMsg.contains("SyntaxCheckException")); + for (String msg: messages) { + assertTrue(errorMsg.contains(msg)); + } + } catch (IOException e) { + throw new IllegalStateException("Unexpected exception raised for query: " + query); + } + } + + private void queryShouldThrowSemanticException(String query, String... messages) { + try { + executeQuery(query); + fail("Expected to throw SemanticCheckException, but none was thrown for query: " + query); + } catch (ResponseException e) { + String errorMsg = e.getMessage(); + assertTrue(errorMsg.contains("SemanticCheckException")); + for (String msg : messages) { + assertTrue(errorMsg.contains(msg)); + } + } catch (IOException e) { + throw new IllegalStateException("Unexpected exception raised for query: " + query); + } + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/RenameCommandIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/RenameCommandIT.java new file mode 100644 index 0000000000..205a5e7672 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/RenameCommandIT.java @@ -0,0 +1,61 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.columnName; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.columnPattern; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyColumn; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.Ignore; +import org.junit.jupiter.api.Test; + +public class RenameCommandIT extends PPLIntegTestCase { + + @Override + public void init() throws IOException { + loadIndex(Index.ACCOUNT); + } + + @Test + public void testRenameOneField() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | fields firstname | rename firstname as first_name", + TEST_INDEX_ACCOUNT)); + verifyColumn(result, columnName("first_name")); + } + + @Test + public void testRenameMultiField() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | fields firstname, age | rename firstname as FIRSTNAME, age as AGE", + TEST_INDEX_ACCOUNT)); + verifyColumn(result, columnName("FIRSTNAME"), columnName("AGE")); + } + + @Ignore("Wildcard is unsupported yet") + @Test + public void testRenameWildcardFields() throws IOException { + JSONObject result = executeQuery("source=" + TEST_INDEX_ACCOUNT + " | rename %name as %NAME"); + verifyColumn(result, columnPattern(".*name$")); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/ResourceMonitorIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/ResourceMonitorIT.java new file mode 100644 index 0000000000..0514bb4320 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/ResourceMonitorIT.java @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_DOG; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.columnName; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyColumn; + +import java.io.IOException; +import org.elasticsearch.client.ResponseException; +import org.hamcrest.Matchers; +import org.json.JSONObject; +import org.junit.Test; + +public class ResourceMonitorIT extends PPLIntegTestCase { + + @Override + public void init() throws IOException { + loadIndex(Index.DOG); + } + + @Test + public void queryExceedResourceLimitShouldFail() throws IOException { + // update opendistro.ppl.query.memory_limit to 1% + updateClusterSettings( + new ClusterSetting("persistent", "opendistro.ppl.query.memory_limit", "1%")); + String query = String.format("search source=%s age=20", TEST_INDEX_DOG); + + ResponseException exception = + expectThrows(ResponseException.class, () -> executeQuery(query)); + assertEquals(500, exception.getResponse().getStatusLine().getStatusCode()); + assertThat(exception.getMessage(), Matchers.containsString("resource is not enough to run the" + + " query, quit.")); + + // update opendistro.ppl.query.memory_limit to default value 85% + updateClusterSettings( + new ClusterSetting("persistent", "opendistro.ppl.query.memory_limit", "85%")); + JSONObject result = executeQuery(String.format("search source=%s", TEST_INDEX_DOG)); + verifyColumn(result, columnName("dog_name"), columnName("holdersName"), columnName("age")); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/SearchCommandIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/SearchCommandIT.java new file mode 100644 index 0000000000..6bdad4ed10 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/SearchCommandIT.java @@ -0,0 +1,71 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl; + +import org.elasticsearch.client.ResponseException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_DOG; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.columnName; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.rows; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyColumn; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyDataRows; + +public class SearchCommandIT extends PPLIntegTestCase { + + @Override + public void init() throws IOException { + loadIndex(Index.BANK); + loadIndex(Index.DOG); + } + + @Test + public void testSearchAllFields() throws IOException { + JSONObject result = executeQuery(String.format("search source=%s", TEST_INDEX_DOG)); + verifyColumn(result, columnName("dog_name"), columnName("holdersName"), columnName("age")); + } + + @Test + public void testSearchCommandWithoutSearchKeyword() throws IOException { + assertEquals( + executeQueryToString(String.format("search source=%s", TEST_INDEX_BANK)), + executeQueryToString(String.format("source=%s", TEST_INDEX_BANK))); + } + + @Test + public void testSearchCommandWithLogicalExpression() throws IOException { + JSONObject result = + executeQuery( + String.format( + "search source=%s firstname='Hattie' | fields firstname", TEST_INDEX_BANK)); + verifyDataRows(result, rows("Hattie")); + } + + @Test + public void searchCommandWithoutSourceShouldFailToParse() throws IOException { + try { + executeQuery("search firstname='Hattie'"); + fail(); + } catch (ResponseException e) { + assertTrue(e.getMessage().contains("RuntimeException")); + assertTrue(e.getMessage().contains("Failed to parse query due to offending symbol")); + } + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/SortCommandIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/SortCommandIT.java new file mode 100644 index 0000000000..71f0bba7dd --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/SortCommandIT.java @@ -0,0 +1,87 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl; + +import org.json.JSONObject; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_DOG; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.rows; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyOrder; + +public class SortCommandIT extends PPLIntegTestCase { + + @Override + public void init() throws IOException { + loadIndex(Index.BANK); + loadIndex(Index.BANK_WITH_NULL_VALUES); + loadIndex(Index.DOG); + } + + @Test + public void testSortCommand() throws IOException { + JSONObject result = + executeQuery(String.format("source=%s | sort age | fields age", TEST_INDEX_BANK)); + verifyOrder(result, rows(28), rows(32), rows(33), rows(34), rows(36), rows(36), rows(39)); + } + + @Ignore("Order with duplicated value") + @Test + public void testSortWithNullValue() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | sort balance | fields firstname, balance", + TEST_INDEX_BANK_WITH_NULL_VALUES)); + verifyOrder( + result, + rows("Hattie"), + rows("Elinor"), + rows("Virginia"), + rows("Dale", 4180), + rows("Nanette", 32838), + rows("Amber JOHnny", 39225), + rows("Dillard", 48086)); + } + + @Test + public void testSortStringField() throws IOException { + JSONObject result = + executeQuery(String.format("source=%s | sort lastname | fields lastname", TEST_INDEX_BANK)); + verifyOrder( + result, + rows("Adams"), + rows("Ayala"), + rows("Bates"), + rows("Bond"), + rows("Duke Willmington"), + rows("Mcpherson"), + rows("Ratliff")); + } + + @Test + public void testSortMultipleFields() throws IOException { + JSONObject result = + executeQuery( + String.format("source=%s | sort dog_name, age | fields dog_name, age", TEST_INDEX_DOG)); + verifyOrder(result, rows("rex", 2), rows("snoopy", 4)); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/StandaloneIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/StandaloneIT.java new file mode 100644 index 0000000000..b017da6f63 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/StandaloneIT.java @@ -0,0 +1,125 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl; + +import static com.amazon.opendistroforelasticsearch.sql.protocol.response.format.JsonResponseFormatter.Style.PRETTY; + +import com.amazon.opendistroforelasticsearch.sql.common.response.ResponseListener; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchRestClient; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor.ElasticsearchExecutionEngine; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor.protector.ElasticsearchExecutionProtector; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.ElasticsearchStorageEngine; +import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine; +import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.QueryResponse; +import com.amazon.opendistroforelasticsearch.sql.monitor.AlwaysHealthyMonitor; +import com.amazon.opendistroforelasticsearch.sql.ppl.config.PPLServiceConfig; +import com.amazon.opendistroforelasticsearch.sql.ppl.domain.PPLQueryRequest; +import com.amazon.opendistroforelasticsearch.sql.protocol.response.QueryResult; +import com.amazon.opendistroforelasticsearch.sql.protocol.response.format.SimpleJsonResponseFormatter; +import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; +import java.io.IOException; +import java.util.concurrent.atomic.AtomicReference; +import org.elasticsearch.client.Node; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; + +/** + * Run PPL with query engine outside Elasticsearch cluster. This IT doesn't require our plugin + * installed actually. The client application, ex. JDBC driver, needs to initialize all components + * itself required by ppl service. + */ +public class StandaloneIT extends PPLIntegTestCase { + + private RestHighLevelClient restClient; + + private PPLService pplService; + + @Override + public void init() { + restClient = + new RestHighLevelClient(RestClient.builder(client().getNodes().toArray(new Node[0]))); + + ElasticsearchClient client = new ElasticsearchRestClient(restClient); + AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); + context.registerBean(StorageEngine.class, + () -> new ElasticsearchStorageEngine(client)); + context.registerBean(ExecutionEngine.class, () -> new ElasticsearchExecutionEngine(client, + new ElasticsearchExecutionProtector(new AlwaysHealthyMonitor()))); + context.register(PPLServiceConfig.class); + context.refresh(); + + pplService = context.getBean(PPLService.class); + } + + @AfterEach + public void tearDown() throws Exception { + restClient.close(); + super.tearDown(); + } + + @Test + public void testSourceFieldQuery() throws IOException { + Request request1 = new Request("PUT", "/test/_doc/1?refresh=true"); + request1.setJsonEntity("{\"name\": \"hello\", \"age\": 20}"); + client().performRequest(request1); + Request request2 = new Request("PUT", "/test/_doc/2?refresh=true"); + request2.setJsonEntity("{\"name\": \"world\", \"age\": 30}"); + client().performRequest(request2); + + String actual = executeByStandaloneQueryEngine("source=test | fields name"); + assertEquals( + "{\n" + + " \"schema\": [{\n" + + " \"name\": \"name\",\n" + + " \"type\": \"string\"\n" + + " }],\n" + + " \"total\": 2,\n" + + " \"datarows\": [\n" + + " [\"hello\"],\n" + + " [\"world\"]\n" + + " ],\n" + + " \"size\": 2\n" + + "}", + actual); + } + + private String executeByStandaloneQueryEngine(String query) { + AtomicReference actual = new AtomicReference<>(); + pplService.execute( + new PPLQueryRequest(query, null), + new ResponseListener() { + + @Override + public void onResponse(QueryResponse response) { + QueryResult result = new QueryResult(response.getResults()); + String json = new SimpleJsonResponseFormatter(PRETTY).format(result); + actual.set(json); + } + + @Override + public void onFailure(Exception e) { + throw new IllegalStateException("Exception happened during execution", e); + } + }); + return actual.get(); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/StatsCommandIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/StatsCommandIT.java new file mode 100644 index 0000000000..819f752e22 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/StatsCommandIT.java @@ -0,0 +1,101 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; + +import java.io.IOException; +import org.junit.jupiter.api.Test; + +public class StatsCommandIT extends PPLIntegTestCase { + + @Override + public void init() throws IOException { + loadIndex(Index.ACCOUNT); + } + + @Test + public void testStatsAvg() throws IOException { + String result = + executeQueryToString(String.format("source=%s | stats avg(age)", TEST_INDEX_ACCOUNT)); + assertEquals( + "{\n" + + " \"schema\": [{\n" + + " \"name\": \"avg(age)\",\n" + + " \"type\": \"double\"\n" + + " }],\n" + + " \"total\": 1,\n" + + " \"datarows\": [[30.171]],\n" + + " \"size\": 1\n" + + "}\n", + result); + } + + @Test + public void testStatsSum() throws IOException { + String result = + executeQueryToString(String.format("source=%s | stats sum(balance)", TEST_INDEX_ACCOUNT)); + assertEquals( + "{\n" + + " \"schema\": [{\n" + + " \"name\": \"sum(balance)\",\n" + + " \"type\": \"long\"\n" + + " }],\n" + + " \"total\": 1,\n" + + " \"datarows\": [[25714837]],\n" + + " \"size\": 1\n" + + "}\n", + result); + } + + @Test + public void testStatsCount() throws IOException { + String result = + executeQueryToString( + String.format("source=%s | stats count(account_number)", TEST_INDEX_ACCOUNT)); + assertEquals( + "{\n" + + " \"schema\": [{\n" + + " \"name\": \"count(account_number)\",\n" + + " \"type\": \"integer\"\n" + + " }],\n" + + " \"total\": 1,\n" + + " \"datarows\": [[1000]],\n" + + " \"size\": 1\n" + + "}\n", + result); + } + + // TODO: each stats aggregate function should be tested here when implemented + + @Test + public void testStatsNested() throws IOException { + String result = + executeQueryToString( + String.format("source=%s | stats avg(abs(age)*2) as AGE", TEST_INDEX_ACCOUNT)); + assertEquals( + "{\n" + + " \"schema\": [{\n" + + " \"name\": \"AGE\",\n" + + " \"type\": \"double\"\n" + + " }],\n" + + " \"total\": 1,\n" + + " \"datarows\": [[60.342]],\n" + + " \"size\": 1\n" + + "}\n", + result); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/WhereCommandIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/WhereCommandIT.java new file mode 100644 index 0000000000..cc74018d1c --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/WhereCommandIT.java @@ -0,0 +1,63 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl; + +import org.json.JSONObject; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.rows; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyDataRows; + +public class WhereCommandIT extends PPLIntegTestCase { + + @Override + public void init() throws IOException { + loadIndex(Index.ACCOUNT); + } + + @Test + public void testWhereWithLogicalExpr() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | fields firstname | where firstname='Amber' | fields firstname", + TEST_INDEX_ACCOUNT)); + verifyDataRows(result, rows("Amber")); + } + + @Test + public void testWhereWithMultiLogicalExpr() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s " + + "| where firstname='Amber' lastname='Duke' age=32 " + + "| fields firstname, lastname, age", + TEST_INDEX_ACCOUNT)); + verifyDataRows(result, rows("Amber", "Duke", 32)); + } + + @Test + public void testWhereEquivalentSortCommand() throws IOException { + assertEquals( + executeQueryToString( + String.format("source=%s | where firstname='Amber'", TEST_INDEX_ACCOUNT)), + executeQueryToString(String.format("source=%s firstname='Amber'", TEST_INDEX_ACCOUNT))); + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/ExpressionIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/ExpressionIT.java new file mode 100644 index 0000000000..46a0a702a4 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/ExpressionIT.java @@ -0,0 +1,106 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.featureValueOf; +import static org.hamcrest.Matchers.is; + +import com.amazon.opendistroforelasticsearch.sql.legacy.RestIntegTestCase; +import com.amazon.opendistroforelasticsearch.sql.util.TestUtils; +import java.io.IOException; +import java.util.Locale; +import java.util.function.Function; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +/** + * Integration test for different type of expressions such as literals, arithmetic, predicate + * and function expression. Since comparison test in {@link SQLCorrectnessIT} is enforced, + * this kind of manual written IT class will be focused on anomaly case test. + */ +public class ExpressionIT extends RestIntegTestCase { + + @Rule + public ExpectedException exceptionRule = ExpectedException.none(); + + @Override + protected void init() throws Exception { + super.init(); + TestUtils.enableNewQueryEngine(client()); + } + + @Test + public void testDivideZeroExpression() throws Exception { + expectResponseException().hasStatusCode(500) //TODO: should be client error code 400? + .containsMessage("\"reason\": \"/ by zero\"") + .containsMessage("\"type\": \"ArithmeticException\"") + .whenExecute("SELECT 5 / (1 - 1)"); + } + + public ResponseExceptionAssertion expectResponseException() { + return new ResponseExceptionAssertion(exceptionRule); + } + + /** + * Response exception assertion helper to assert property value in ES ResponseException + * and Response inside. This serves as syntax sugar to improve the readability of test + * code. + */ + private static class ResponseExceptionAssertion { + private final ExpectedException exceptionRule; + + private ResponseExceptionAssertion(ExpectedException exceptionRule) { + this.exceptionRule = exceptionRule; + + exceptionRule.expect(ResponseException.class); + } + + ResponseExceptionAssertion hasStatusCode(int expected) { + exceptionRule.expect(featureValueOf("statusCode", is(expected), + (Function) e -> + e.getResponse().getStatusLine().getStatusCode())); + return this; + } + + ResponseExceptionAssertion containsMessage(String expected) { + exceptionRule.expectMessage(expected); + return this; + } + + void whenExecute(String query) throws Exception { + executeQuery(query); + } + } + + private static Response executeQuery(String query) throws IOException { + Request request = new Request("POST", QUERY_API_ENDPOINT); + request.setJsonEntity(String.format(Locale.ROOT, "{\n" + " \"query\": \"%s\"\n" + "}", query)); + + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + request.setOptions(restOptionsBuilder); + + return client().performRequest(request); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLCorrectnessIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLCorrectnessIT.java new file mode 100644 index 0000000000..7ad193b985 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLCorrectnessIT.java @@ -0,0 +1,76 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql; + +import com.amazon.opendistroforelasticsearch.sql.util.TestUtils; +import com.google.common.io.Resources; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.function.Function; +import org.junit.Test; + +/** + * SQL integration test automated by comparison test framework. + */ +public class SQLCorrectnessIT extends SQLIntegTestCase { + + private static final String ROOT_DIR = "correctness/"; + private static final String[] EXPR_TEST_DIR = { "expressions" }; + private static final String[] QUERY_TEST_DIR = { "queries"/*, "bugfixes"*/ }; //TODO: skip bugfixes folder for now since it fails + + @Override + protected void init() throws Exception { + super.init(); + TestUtils.enableNewQueryEngine(client()); + } + + @Test + public void runAllTests() throws Exception { + verifyQueries(EXPR_TEST_DIR, expr -> "SELECT " + expr); + verifyQueries(QUERY_TEST_DIR, Function.identity()); + } + + /** + * Verify queries in files in directories with a converter to preprocess query. + * For example, for expressions it is converted to a SELECT clause before testing. + */ + @SuppressWarnings("UnstableApiUsage") + private void verifyQueries(String[] dirs, Function converter) throws Exception { + for (String dir : dirs) { + Path dirPath = Paths.get(Resources.getResource(ROOT_DIR + dir).toURI()); + Files.walk(dirPath) + .filter(Files::isRegularFile) + .forEach(file -> verifyQueries(file, converter)); + } + } + + private void verifyQueries(Path file, Function converter) { + try { + String[] queries = Files.lines(file) + .map(converter) + .toArray(String[]::new); + verify(queries); + } catch (IOException e) { + throw new IllegalStateException("Failed to read file: " + file, e); + } + } + + + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLIntegTestCase.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLIntegTestCase.java new file mode 100644 index 0000000000..faef8f713f --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLIntegTestCase.java @@ -0,0 +1,112 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql; + +import static java.util.Collections.emptyMap; + +import com.amazon.opendistroforelasticsearch.sql.correctness.TestConfig; +import com.amazon.opendistroforelasticsearch.sql.correctness.report.TestReport; +import com.amazon.opendistroforelasticsearch.sql.correctness.report.TestSummary; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.ComparisonTest; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection.DBConnection; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection.ESConnection; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection.JDBCConnection; +import com.amazon.opendistroforelasticsearch.sql.correctness.testset.TestDataSet; +import com.amazon.opendistroforelasticsearch.sql.correctness.testset.TestQuerySet; +import com.amazon.opendistroforelasticsearch.sql.legacy.RestIntegTestCase; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.json.JSONObject; +import org.junit.AfterClass; +import org.junit.Assert; + +/** + * SQL integration test base class. This is very similar to CorrectnessIT though + * enforce the success of all tests rather than report failures only. + */ +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public abstract class SQLIntegTestCase extends RestIntegTestCase { + + /** + * Comparison test runner shared by all methods in this IT class. + */ + private static ComparisonTest runner; + + @Override + protected void init() throws Exception { + if (runner != null) { + return; + } + + TestConfig config = new TestConfig(emptyMap()); + runner = new ComparisonTest(getESConnection(), + getOtherDBConnections(config)); + + runner.connect(); + for (TestDataSet dataSet : config.getTestDataSets()) { + runner.loadData(dataSet); + } + } + + /** + * Clean up test data and close other database connection. + */ + @AfterClass + public static void cleanUp() { + try { + TestConfig config = new TestConfig(emptyMap()); + for (TestDataSet dataSet : config.getTestDataSets()) { + runner.cleanUp(dataSet); + } + + runner.close(); + } finally { + runner = null; + } + } + + /** + * Execute the given queries and compare result with other database. + * The queries will be considered as one test batch. + */ + protected void verify(String... queries) { + TestReport result = runner.verify(new TestQuerySet(queries)); + TestSummary summary = result.getSummary(); + Assert.assertEquals(StringUtils.format( + "Comparison test failed on queries: %s", new JSONObject(result).toString(2)), + 0, summary.getFailure()); + } + + /** + * Use Elasticsearch cluster initialized by ES Gradle task. + */ + private DBConnection getESConnection() { + String esHost = client().getNodes().get(0).getHost().toString(); + return new ESConnection("jdbc:elasticsearch://" + esHost, client()); + } + + /** + * Create database connection with database name and connect URL. + */ + private DBConnection[] getOtherDBConnections(TestConfig config) { + return config.getOtherDbConnectionNameAndUrls() + .entrySet().stream() + .map(e -> new JDBCConnection(e.getKey(), e.getValue())) + .toArray(DBConnection[]::new); + } + +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/MatcherUtils.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/MatcherUtils.java new file mode 100644 index 0000000000..0668b06641 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/MatcherUtils.java @@ -0,0 +1,307 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.util; + +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.arrayContaining; +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsInRelativeOrder; +import static org.hamcrest.Matchers.emptyArray; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasItems; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; + +import com.google.common.base.Strings; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.hamcrest.Description; +import org.hamcrest.FeatureMatcher; +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeMatcher; +import org.json.JSONArray; +import org.json.JSONObject; + +public class MatcherUtils { + + /** + * Assert field value in object by a custom matcher and getter to access the field. + * + * @param name description + * @param subMatcher sub-matcher for field + * @param getter getter function to access the field + * @param type of outer object + * @param type of inner field + * @return matcher + */ + public static FeatureMatcher featureValueOf(String name, + Matcher subMatcher, + Function getter) { + return new FeatureMatcher(subMatcher, name, name) { + @Override + protected U featureValueOf(T actual) { + return getter.apply(actual); + } + }; + } + + @SafeVarargs + public static Matcher hits(Matcher... hitMatchers) { + if (hitMatchers.length == 0) { + return featureValueOf("SearchHits", emptyArray(), SearchHits::getHits); + } + return featureValueOf("SearchHits", arrayContainingInAnyOrder(hitMatchers), + SearchHits::getHits); + } + + @SafeVarargs + public static Matcher hitsInOrder(Matcher... hitMatchers) { + if (hitMatchers.length == 0) { + return featureValueOf("SearchHits", emptyArray(), SearchHits::getHits); + } + return featureValueOf("SearchHits", arrayContaining(hitMatchers), SearchHits::getHits); + } + + @SuppressWarnings("unchecked") + public static Matcher hit(Matcher>... entryMatchers) { + return featureValueOf("SearchHit", allOf(entryMatchers), SearchHit::getSourceAsMap); + } + + @SuppressWarnings("unchecked") + public static Matcher> kv(String key, Object value) { + // Use raw type to avoid generic type problem from Matcher> to Matcher + return (Matcher) hasEntry(key, value); + } + + public static Matcher hitAny(String query, Matcher... matcher) { + return featureValueOf("SearchHits", hasItems(matcher), actual -> { + JSONArray array = (JSONArray) (actual.query(query)); + List results = new ArrayList<>(array.length()); + for (Object element : array) { + results.add((JSONObject) element); + } + return results; + }); + } + + public static Matcher hitAny(Matcher... matcher) { + return hitAny("/hits/hits", matcher); + } + + public static Matcher hitAll(Matcher... matcher) { + return featureValueOf("SearchHits", containsInAnyOrder(matcher), actual -> { + JSONArray array = (JSONArray) (actual.query("/hits/hits")); + List results = new ArrayList<>(array.length()); + for (Object element : array) { + results.add((JSONObject) element); + } + return results; + }); + } + + public static Matcher kvString(String key, Matcher matcher) { + return featureValueOf("Json Match", matcher, actual -> (String) actual.query(key)); + } + + public static Matcher kvDouble(String key, Matcher matcher) { + return featureValueOf("Json Match", matcher, actual -> (Double) actual.query(key)); + } + + public static Matcher kvInt(String key, Matcher matcher) { + return featureValueOf("Json Match", matcher, actual -> (Integer) actual.query(key)); + } + + @SafeVarargs + public static void verifySchema(JSONObject response, Matcher... matchers) { + verify(response.getJSONArray("schema"), matchers); + } + + @SafeVarargs + public static void verifyDataRows(JSONObject response, Matcher... matchers) { + verify(response.getJSONArray("datarows"), matchers); + } + + @SafeVarargs + public static void verifyColumn(JSONObject response, Matcher... matchers) { + verify(response.getJSONArray("schema"), matchers); + } + + @SafeVarargs + public static void verifyOrder(JSONObject response, Matcher... matchers) { + verifyOrder(response.getJSONArray("datarows"), matchers); + } + + @SafeVarargs + @SuppressWarnings("unchecked") + public static void verifyDataRowsInOrder(JSONObject response, Matcher... matchers) { + verifyInOrder(response.getJSONArray("datarows"), matchers); + } + + @SuppressWarnings("unchecked") + public static void verify(JSONArray array, Matcher... matchers) { + List objects = new ArrayList<>(); + array.iterator().forEachRemaining(o -> objects.add((T) o)); + assertEquals(matchers.length, objects.size()); + assertThat(objects, containsInAnyOrder(matchers)); + } + + @SafeVarargs + @SuppressWarnings("unchecked") + public static void verifyInOrder(JSONArray array, Matcher... matchers) { + List objects = new ArrayList<>(); + array.iterator().forEachRemaining(o -> objects.add((T) o)); + assertEquals(matchers.length, objects.size()); + assertThat(objects, contains(matchers)); + } + + @SuppressWarnings("unchecked") + public static void verifySome(JSONArray array, Matcher... matchers) { + List objects = new ArrayList<>(); + array.iterator().forEachRemaining(o -> objects.add((T) o)); + + assertThat(matchers.length, greaterThan(0)); + for (Matcher matcher : matchers) { + assertThat(objects, hasItems(matcher)); + } + } + + @SafeVarargs + public static void verifyOrder(JSONArray array, Matcher... matchers) { + List objects = new ArrayList<>(); + array.iterator().forEachRemaining(o -> objects.add((T) o)); + assertEquals(matchers.length, objects.size()); + assertThat(objects, containsInRelativeOrder(matchers)); + } + + public static TypeSafeMatcher schema(String expectedName, String expectedAlias, + String expectedType) { + return new TypeSafeMatcher() { + @Override + public void describeTo(Description description) { + description.appendText( + String + .format("(name=%s, alias=%s, type=%s)", expectedName, expectedAlias, expectedType)); + } + + @Override + protected boolean matchesSafely(JSONObject jsonObject) { + String actualName = (String) jsonObject.query("/name"); + String actualAlias = (String) jsonObject.query("/alias"); + String actualType = (String) jsonObject.query("/type"); + return expectedName.equals(actualName) && + (Strings.isNullOrEmpty(actualAlias) && Strings.isNullOrEmpty(expectedAlias) || + expectedAlias.equals(actualAlias)) && + expectedType.equals(actualType); + } + }; + } + + public static TypeSafeMatcher rows(Object... expectedObjects) { + return new TypeSafeMatcher() { + @Override + public void describeTo(Description description) { + description.appendText(String.join(",", Arrays.asList(expectedObjects).toString())); + } + + @Override + protected boolean matchesSafely(JSONArray array) { + List actualObjects = new ArrayList<>(); + array.iterator().forEachRemaining(actualObjects::add); + return Arrays.asList(expectedObjects).equals(actualObjects); + } + }; + } + + public static TypeSafeMatcher columnPattern(String regex) { + return new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(JSONObject jsonObject) { + return ((String) jsonObject.query("/name")).matches(regex); + } + + @Override + public void describeTo(Description description) { + description.appendText(String.format("(column_pattern=%s)", regex)); + } + }; + } + + public static TypeSafeMatcher columnName(String name) { + return new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(JSONObject jsonObject) { + return jsonObject.query("/name").equals(name); + } + + @Override + public void describeTo(Description description) { + description.appendText(String.format("(name=%s)", name)); + } + }; + } + + + /** + * Tests if a string is equal to another string, ignore the case and whitespace. + */ + public static class IsEqualIgnoreCaseAndWhiteSpace extends TypeSafeMatcher { + private final String string; + + public IsEqualIgnoreCaseAndWhiteSpace(String string) { + if (string == null) { + throw new IllegalArgumentException("Non-null value required"); + } + this.string = string; + } + + @Override + public boolean matchesSafely(String item) { + return ignoreCase(ignoreSpaces(string)).equals(ignoreCase(ignoreSpaces(item))); + } + + @Override + public void describeMismatchSafely(String item, Description mismatchDescription) { + mismatchDescription.appendText("was ").appendValue(item); + } + + @Override + public void describeTo(Description description) { + description.appendText("a string equal to ") + .appendValue(string) + .appendText(" ignore case and white space"); + } + + public String ignoreSpaces(String toBeStripped) { + return toBeStripped.replaceAll("\\s+", "").trim(); + } + + public String ignoreCase(String toBeLower) { + return toBeLower.toLowerCase(); + } + + public static Matcher equalToIgnoreCaseAndWhiteSpace(String expectedString) { + return new IsEqualIgnoreCaseAndWhiteSpace(expectedString); + } + } +} diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/TestUtils.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/TestUtils.java new file mode 100644 index 0000000000..f68dd74456 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/TestUtils.java @@ -0,0 +1,871 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.util; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlSettingsAction.SETTINGS_API_ENDPOINT; +import static com.google.common.base.Strings.isNullOrEmpty; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.RestStatus; +import org.json.JSONObject; +import org.junit.Assert; + +public class TestUtils { + + /** + * Create test index by REST client. + * + * @param client client connection + * @param indexName test index name + * @param mapping test index mapping or null if no predefined mapping + */ + public static void createIndexByRestClient(RestClient client, String indexName, String mapping) { + Request request = new Request("PUT", "/" + indexName); + if (!isNullOrEmpty(mapping)) { + request.setJsonEntity(mapping); + } + performRequest(client, request); + } + + /** + * https://github.com/elastic/elasticsearch/pull/49959 + * Deprecate creation of dot-prefixed index names except for hidden and system indices. + * Create hidden index by REST client. + * + * @param client client connection + * @param indexName test index name + * @param mapping test index mapping or null if no predefined mapping + */ + public static void createHiddenIndexByRestClient(RestClient client, String indexName, + String mapping) { + Request request = new Request("PUT", "/" + indexName); + JSONObject jsonObject = isNullOrEmpty(mapping) ? new JSONObject() : new JSONObject(mapping); + jsonObject.put("settings", new JSONObject("{\"index\":{\"hidden\":true}}")); + request.setJsonEntity(jsonObject.toString()); + + performRequest(client, request); + } + + /** + * Check if index already exists by ES index exists API which returns: + * 200 - specified indices or aliases exist + * 404 - one or more indices specified or aliases do not exist + * + * @param client client connection + * @param indexName index name + * @return true for index exist + */ + public static boolean isIndexExist(RestClient client, String indexName) { + try { + Response response = client.performRequest(new Request("HEAD", "/" + indexName)); + return (response.getStatusLine().getStatusCode() == 200); + } catch (IOException e) { + throw new IllegalStateException("Failed to perform request", e); + } + } + + /** + * Load test data set by REST client. + * + * @param client client connection + * @param indexName index name + * @param dataSetFilePath file path of test data set + * @throws IOException + */ + public static void loadDataByRestClient(RestClient client, String indexName, + String dataSetFilePath) throws IOException { + Path path = Paths.get(getResourceFilePath(dataSetFilePath)); + Request request = new Request("POST", "/" + indexName + "/_bulk?refresh=true"); + request.setJsonEntity(new String(Files.readAllBytes(path))); + performRequest(client, request); + } + + /** + * Perform a request by REST client. + * + * @param client client connection + * @param request request object + */ + public static Response performRequest(RestClient client, Request request) { + try { + Response response = client.performRequest(request); + int status = response.getStatusLine().getStatusCode(); + if (status >= 400) { + throw new IllegalStateException("Failed to perform request. Error code: " + status); + } + return response; + } catch (IOException e) { + throw new IllegalStateException("Failed to perform request", e); + } + } + + public static String getAccountIndexMapping() { + return "{ \"mappings\": {" + + " \"properties\": {\n" + + " \"gender\": {\n" + + " \"type\": \"text\",\n" + + " \"fielddata\": true\n" + + " }," + + " \"address\": {\n" + + " \"type\": \"text\",\n" + + " \"fielddata\": true\n" + + " }," + + " \"firstname\": {\n" + + " \"type\": \"text\",\n" + + " \"fielddata\": true,\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }" + + " }" + + " }," + + " \"lastname\": {\n" + + " \"type\": \"text\",\n" + + " \"fielddata\": true,\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }" + + " }" + + " }," + + " \"state\": {\n" + + " \"type\": \"text\",\n" + + " \"fielddata\": true,\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }" + + " }" + + " }" + + " }" + + " }" + + "}"; + } + + public static String getPhraseIndexMapping() { + return "{ \"mappings\": {" + + " \"properties\": {\n" + + " \"phrase\": {\n" + + " \"type\": \"text\",\n" + + " \"store\": true\n" + + " }" + + " }" + + " }" + + "}"; + } + + public static String getDogIndexMapping() { + return "{ \"mappings\": {" + + " \"properties\": {\n" + + " \"dog_name\": {\n" + + " \"type\": \"text\",\n" + + " \"fielddata\": true\n" + + " }" + + " }" + + " }" + + "}"; + } + + public static String getDogs2IndexMapping() { + return "{ \"mappings\": {" + + " \"properties\": {\n" + + " \"dog_name\": {\n" + + " \"type\": \"text\",\n" + + " \"fielddata\": true\n" + + " },\n" + + " \"holdersName\": {\n" + + " \"type\": \"keyword\"\n" + + " }" + + " }" + + " }" + + "}"; + } + + public static String getDogs3IndexMapping() { + return "{ \"mappings\": {" + + " \"properties\": {\n" + + " \"holdersName\": {\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"color\": {\n" + + " \"type\": \"text\"\n" + + " }" + + " }" + + " }" + + "}"; + } + + public static String getPeople2IndexMapping() { + return "{ \"mappings\": {" + + " \"properties\": {\n" + + " \"firstname\": {\n" + + " \"type\": \"keyword\"\n" + + " }" + + " }" + + " }" + + "}"; + } + + public static String getGameOfThronesIndexMapping() { + return "{ \"mappings\": { " + + " \"properties\": {\n" + + " \"nickname\": {\n" + + " \"type\":\"text\", " + + " \"fielddata\":true" + + " },\n" + + " \"name\": {\n" + + " \"properties\": {\n" + + " \"firstname\": {\n" + + " \"type\": \"text\",\n" + + " \"fielddata\": true\n" + + " },\n" + + " \"lastname\": {\n" + + " \"type\": \"text\",\n" + + " \"fielddata\": true\n" + + " },\n" + + " \"ofHerName\": {\n" + + " \"type\": \"integer\"\n" + + " },\n" + + " \"ofHisName\": {\n" + + " \"type\": \"integer\"\n" + + " }\n" + + " }\n" + + " },\n" + + " \"house\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + + " },\n" + + " \"gender\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + + " }" + + "} } }"; + } + + // System + + public static String getOdbcIndexMapping() { + return "{\n" + + "\t\"mappings\" :{\n" + + "\t\t\"properties\":{\n" + + "\t\t\t\"odbc_time\":{\n" + + "\t\t\t\t\"type\":\"date\",\n" + + "\t\t\t\t\"format\": \"'{ts' ''yyyy-MM-dd HH:mm:ss.SSS'''}'\"\n" + + "\t\t\t},\n" + + "\t\t\t\"docCount\":{\n" + + "\t\t\t\t\"type\":\"text\"\n" + + "\t\t\t}\n" + + "\t\t}\n" + + "\t}\n" + + "}"; + } + + public static String getLocationIndexMapping() { + return "{\n" + + "\t\"mappings\" :{\n" + + "\t\t\"properties\":{\n" + + "\t\t\t\"place\":{\n" + + "\t\t\t\t\"type\":\"geo_shape\"\n" + + //"\t\t\t\t\"tree\": \"quadtree\",\n" + // Field tree and precision are deprecated in ES + //"\t\t\t\t\"precision\": \"10km\"\n" + + "\t\t\t},\n" + + "\t\t\t\"center\":{\n" + + "\t\t\t\t\"type\":\"geo_point\"\n" + + "\t\t\t},\n" + + "\t\t\t\"description\":{\n" + + "\t\t\t\t\"type\":\"text\"\n" + + "\t\t\t}\n" + + "\t\t}\n" + + "\t}\n" + + "}"; + } + + public static String getEmployeeNestedTypeIndexMapping() { + return "{\n" + + " \"mappings\": {\n" + + " \"properties\": {\n" + + " \"comments\": {\n" + + " \"type\": \"nested\",\n" + + " \"properties\": {\n" + + " \"date\": {\n" + + " \"type\": \"date\"\n" + + " },\n" + + " \"likes\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"message\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"id\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"projects\": {\n" + + " \"type\": \"nested\",\n" + + " \"properties\": {\n" + + " \"address\": {\n" + + " \"type\": \"nested\",\n" + + " \"properties\": {\n" + + " \"city\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"state\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " },\n" + + " \"fielddata\": true\n" + + " },\n" + + " \"started_year\": {\n" + + " \"type\": \"long\"\n" + + " }\n" + + " }\n" + + " },\n" + + " \"title\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n"; + } + + + public static String getNestedTypeIndexMapping() { + return "{ \"mappings\": {\n" + + " \"properties\": {\n" + + " \"message\": {\n" + + " \"type\": \"nested\",\n" + + " \"properties\": {\n" + + " \"info\": {\n" + + " \"type\": \"keyword\",\n" + + " \"index\": \"true\"\n" + + " },\n" + + " \"author\": {\n" + + " \"type\": \"keyword\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\" : 256\n" + + " }\n" + + " },\n" + + " \"index\": \"true\"\n" + + " },\n" + + " \"dayOfWeek\": {\n" + + " \"type\": \"long\"\n" + + " }\n" + + " }\n" + + " },\n" + + " \"comment\": {\n" + + " \"type\": \"nested\",\n" + + " \"properties\": {\n" + + " \"data\": {\n" + + " \"type\": \"keyword\",\n" + + " \"index\": \"true\"\n" + + " },\n" + + " \"likes\": {\n" + + " \"type\": \"long\"\n" + + " }\n" + + " }\n" + + " },\n" + + " \"myNum\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"someField\": {\n" + + " \"type\": \"keyword\",\n" + + " \"index\": \"true\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }}"; + } + + public static String getJoinTypeIndexMapping() { + return "{\n" + + " \"mappings\": {\n" + + " \"properties\": {\n" + + " \"join_field\": {\n" + + " \"type\": \"join\",\n" + + " \"relations\": {\n" + + " \"parentType\": \"childrenType\"\n" + + " }\n" + + " },\n" + + " \"parentTile\": {\n" + + " \"index\": \"true\",\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"dayOfWeek\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"author\": {\n" + + " \"index\": \"true\",\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"info\": {\n" + + " \"index\": \"true\",\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + } + + public static String getBankIndexMapping() { + return "{\n" + + " \"mappings\": {\n" + + " \"properties\": {\n" + + " \"account_number\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"address\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"age\": {\n" + + " \"type\": \"integer\"\n" + + " },\n" + + " \"balance\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"birthdate\": {\n" + + " \"type\": \"date\"\n" + + " },\n" + + " \"city\": {\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"email\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"employer\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"firstname\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"gender\": {\n" + + " \"type\": \"text\",\n" + + " \"fielddata\": true\n" + + " }," + + " \"lastname\": {\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"male\": {\n" + + " \"type\": \"boolean\"\n" + + " },\n" + + " \"state\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + } + + public static String getBankWithNullValuesIndexMapping() { + return "{\n" + + " \"mappings\": {\n" + + " \"properties\": {\n" + + " \"account_number\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"address\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"age\": {\n" + + " \"type\": \"integer\"\n" + + " },\n" + + " \"balance\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"gender\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"firstname\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"lastname\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + } + + public static String getOrderIndexMapping() { + return "{\n" + + " \"mappings\": {\n" + + " \"properties\": {\n" + + " \"id\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + } + + public static String getWeblogsIndexMapping() { + return "{\n" + + " \"mappings\": {\n" + + " \"properties\": {\n" + + " \"host\": {\n" + + " \"type\": \"ip\"\n" + + " },\n" + + " \"method\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"url\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"response\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"bytes\": {\n" + + " \"type\": \"text\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + } + + public static String getDateIndexMapping() { + return "{ \"mappings\": {" + + " \"properties\": {\n" + + " \"date_keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }" + + " }" + + " }" + + "}"; + } + + public static String getDateTimeIndexMapping() { + return "{" + + " \"mappings\": {" + + " \"properties\": {" + + " \"birthday\": {" + + " \"type\": \"date\"" + + " }" + + " }" + + " }" + + "}"; + } + + public static String getNestedSimpleIndexMapping() { + return "{" + + " \"mappings\": {" + + " \"properties\": {" + + " \"address\": {" + + " \"type\": \"nested\"," + + " \"properties\": {" + + " \"city\": {" + + " \"type\": \"text\"," + + " \"fields\": {" + + " \"keyword\": {" + + " \"type\": \"keyword\"," + + " \"ignore_above\": 256" + + " }" + + " }" + + " }," + + " \"state\": {" + + " \"type\": \"text\"," + + " \"fields\": {" + + " \"keyword\": {" + + " \"type\": \"keyword\"," + + " \"ignore_above\": 256" + + " }" + + " }" + + " }" + + " }" + + " }," + + " \"age\": {" + + " \"type\": \"long\"" + + " }," + + " \"id\": {" + + " \"type\": \"long\"" + + " }," + + " \"name\": {" + + " \"type\": \"text\"," + + " \"fields\": {" + + " \"keyword\": {" + + " \"type\": \"keyword\"," + + " \"ignore_above\": 256" + + " }" + + " }" + + " }" + + " }" + + " }" + + "}"; + } + + public static void loadBulk(Client client, String jsonPath, String defaultIndex) + throws Exception { + System.out.println(String.format("Loading file %s into elasticsearch cluster", jsonPath)); + String absJsonPath = getResourceFilePath(jsonPath); + + BulkRequest bulkRequest = new BulkRequest(); + try (final InputStream stream = new FileInputStream(absJsonPath); + final Reader streamReader = new InputStreamReader(stream, StandardCharsets.UTF_8); + final BufferedReader br = new BufferedReader(streamReader)) { + + while (true) { + + String actionLine = br.readLine(); + if (actionLine == null || actionLine.trim().isEmpty()) { + break; + } + String sourceLine = br.readLine(); + JSONObject actionJson = new JSONObject(actionLine); + + IndexRequest indexRequest = new IndexRequest(); + indexRequest.index(defaultIndex); + String docType = actionJson.getJSONObject("index").getString("_type"); + indexRequest.type(docType); + if (actionJson.getJSONObject("index").has("_id")) { + String docId = actionJson.getJSONObject("index").getString("_id"); + indexRequest.id(docId); + } + if (actionJson.getJSONObject("index").has("_routing")) { + String routing = actionJson.getJSONObject("index").getString("_routing"); + indexRequest.routing(routing); + } + indexRequest.source(sourceLine, XContentType.JSON); + bulkRequest.add(indexRequest); + } + } + + BulkResponse bulkResponse = client.bulk(bulkRequest).actionGet(); + + if (bulkResponse.hasFailures()) { + throw new Exception("Failed to load test data into index " + defaultIndex + ", " + + bulkResponse.buildFailureMessage()); + } + System.out.println(bulkResponse.getItems().length + " documents loaded."); + // ensure the documents are searchable + client.admin().indices().prepareRefresh(defaultIndex).execute().actionGet(); + } + + public static String getResourceFilePath(String relPath) { + String projectRoot = System.getProperty("project.root", null); + if (projectRoot == null) { + return new File(relPath).getAbsolutePath(); + } else { + return new File(projectRoot + "/" + relPath).getAbsolutePath(); + } + } + + public static String getResponseBody(Response response) throws IOException { + + return getResponseBody(response, false); + } + + public static String getResponseBody(Response response, boolean retainNewLines) + throws IOException { + final StringBuilder sb = new StringBuilder(); + + try (final InputStream is = response.getEntity().getContent(); + final BufferedReader br = new BufferedReader( + new InputStreamReader(is, StandardCharsets.UTF_8))) { + + String line; + while ((line = br.readLine()) != null) { + sb.append(line); + if (retainNewLines) { + sb.append(String.format(Locale.ROOT, "%n")); + } + } + } + return sb.toString(); + } + + public static String fileToString(final String filePathFromProjectRoot, + final boolean removeNewLines) + throws IOException { + + final String absolutePath = getResourceFilePath(filePathFromProjectRoot); + + try (final InputStream stream = new FileInputStream(absolutePath); + final Reader streamReader = new InputStreamReader(stream, StandardCharsets.UTF_8); + final BufferedReader br = new BufferedReader(streamReader)) { + + final StringBuilder stringBuilder = new StringBuilder(); + String line = br.readLine(); + + while (line != null) { + + stringBuilder.append(line); + if (!removeNewLines) { + stringBuilder.append(String.format(Locale.ROOT, "%n")); + } + line = br.readLine(); + } + + return stringBuilder.toString(); + } + } + + /** + * Builds all permutations of the given list of Strings + * + * @param items list of strings to permute + * @return list of permutations + */ + public static List> getPermutations(final List items) { + + if (items.size() > 5) { + throw new IllegalArgumentException("Inefficient test, please refactor"); + } + + final List> result = new LinkedList<>(); + + if (items.isEmpty() || 1 == items.size()) { + + final List onlyElement = new ArrayList<>(); + if (1 == items.size()) { + onlyElement.add(items.get(0)); + } + result.add(onlyElement); + return result; + } + + for (int i = 0; i < items.size(); ++i) { + + final List smallerSet = new ArrayList<>(); + + if (i != 0) { + smallerSet.addAll(items.subList(0, i)); + } + if (i != items.size() - 1) { + smallerSet.addAll(items.subList(i + 1, items.size())); + } + + final String currentItem = items.get(i); + result.addAll(getPermutations(smallerSet).stream().map(smallerSetPermutation -> { + final List permutation = new ArrayList<>(); + permutation.add(currentItem); + permutation.addAll(smallerSetPermutation); + return permutation; + }).collect(Collectors.toCollection(LinkedList::new))); + } + + return result; + } + + /** + * Enable new query engine which is disabled by default for now. + */ + public static void enableNewQueryEngine(RestClient client) throws IOException { + Request request = new Request("PUT", SETTINGS_API_ENDPOINT); + request.setJsonEntity("{\"transient\" : {\"opendistro.sql.engine.new.enabled\" : \"true\"}}"); + + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + request.setOptions(restOptionsBuilder); + + Response response = client.performRequest(request); + Assert.assertEquals(RestStatus.OK, + RestStatus.fromCode(response.getStatusLine().getStatusCode())); + } + +} diff --git a/src/test/resources/.bank.json b/integ-test/src/test/resources/.bank.json similarity index 100% rename from src/test/resources/.bank.json rename to integ-test/src/test/resources/.bank.json diff --git a/src/test/resources/accounts.json b/integ-test/src/test/resources/accounts.json similarity index 100% rename from src/test/resources/accounts.json rename to integ-test/src/test/resources/accounts.json diff --git a/src/test/resources/accounts_temp.json b/integ-test/src/test/resources/accounts_temp.json similarity index 100% rename from src/test/resources/accounts_temp.json rename to integ-test/src/test/resources/accounts_temp.json diff --git a/src/test/resources/bank.json b/integ-test/src/test/resources/bank.json similarity index 100% rename from src/test/resources/bank.json rename to integ-test/src/test/resources/bank.json diff --git a/src/test/resources/bank_for_unquote_test.json b/integ-test/src/test/resources/bank_for_unquote_test.json similarity index 100% rename from src/test/resources/bank_for_unquote_test.json rename to integ-test/src/test/resources/bank_for_unquote_test.json diff --git a/src/test/resources/bank_two.json b/integ-test/src/test/resources/bank_two.json similarity index 100% rename from src/test/resources/bank_two.json rename to integ-test/src/test/resources/bank_two.json diff --git a/src/test/resources/bank_with_null_values.json b/integ-test/src/test/resources/bank_with_null_values.json similarity index 94% rename from src/test/resources/bank_with_null_values.json rename to integ-test/src/test/resources/bank_with_null_values.json index 6eab13677e..09000a201b 100644 --- a/src/test/resources/bank_with_null_values.json +++ b/integ-test/src/test/resources/bank_with_null_values.json @@ -9,6 +9,6 @@ {"index":{"_id":"20"}} {"account_number":20,"firstname":"Elinor","lastname":"Ratliff","age":36,"gender":"M","address":"282 Kings Place"} {"index":{"_id":"25"}} -{"account_number":25,"firstname":"Virginia","lastname":"Ayala","age":39,"gender":"F","address":"171 Putnam Avenue"} +{"account_number":25,"firstname":"Virginia","lastname":"Ayala","age":null, "gender":"F","address":"171 Putnam Avenue"} {"index":{"_id":"32"}} {"account_number":32,"balance":48086,"firstname":"Dillard","lastname":"Mcpherson","age":34,"gender":"F","address":"702 Quentin Street"} diff --git a/integ-test/src/test/resources/correctness/bugfixes/521.txt b/integ-test/src/test/resources/correctness/bugfixes/521.txt new file mode 100644 index 0000000000..72a27b01d7 --- /dev/null +++ b/integ-test/src/test/resources/correctness/bugfixes/521.txt @@ -0,0 +1 @@ +SELECT timestamp, COUNT(*) FROM kibana_sample_data_flights GROUP BY timestamp \ No newline at end of file diff --git a/integ-test/src/test/resources/correctness/expressions/arithmetics.txt b/integ-test/src/test/resources/correctness/expressions/arithmetics.txt new file mode 100644 index 0000000000..64872a4df3 --- /dev/null +++ b/integ-test/src/test/resources/correctness/expressions/arithmetics.txt @@ -0,0 +1,11 @@ +1 + 2 +-1.0 + 1.234 +10 - 5 +1.2 - 3 +5 * -6 +-1 * 2.0 +1 / 2 +-4 / 2.0 +5 % 2 +-5 % 2 +0 % 2 diff --git a/integ-test/src/test/resources/correctness/expressions/functions.txt b/integ-test/src/test/resources/correctness/expressions/functions.txt new file mode 100644 index 0000000000..5b67c88b66 --- /dev/null +++ b/integ-test/src/test/resources/correctness/expressions/functions.txt @@ -0,0 +1,7 @@ +abs(-1) +abs(0) +abs(1) +abs(-1.234) +abs(0.0) +abs(4.321) +abs(abs(-1.2) * -1) \ No newline at end of file diff --git a/integ-test/src/test/resources/correctness/expressions/literals.txt b/integ-test/src/test/resources/correctness/expressions/literals.txt new file mode 100644 index 0000000000..74710d917b --- /dev/null +++ b/integ-test/src/test/resources/correctness/expressions/literals.txt @@ -0,0 +1,4 @@ +1 +true +-4.567 +-123,false diff --git a/integ-test/src/test/resources/correctness/expressions/predicates.txt b/integ-test/src/test/resources/correctness/expressions/predicates.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/test/resources/correctness/kibana_sample_data_ecommerce.csv b/integ-test/src/test/resources/correctness/kibana_sample_data_ecommerce.csv similarity index 100% rename from src/test/resources/correctness/kibana_sample_data_ecommerce.csv rename to integ-test/src/test/resources/correctness/kibana_sample_data_ecommerce.csv diff --git a/src/test/resources/correctness/kibana_sample_data_ecommerce.json b/integ-test/src/test/resources/correctness/kibana_sample_data_ecommerce.json similarity index 100% rename from src/test/resources/correctness/kibana_sample_data_ecommerce.json rename to integ-test/src/test/resources/correctness/kibana_sample_data_ecommerce.json diff --git a/src/test/resources/correctness/kibana_sample_data_flights.csv b/integ-test/src/test/resources/correctness/kibana_sample_data_flights.csv similarity index 100% rename from src/test/resources/correctness/kibana_sample_data_flights.csv rename to integ-test/src/test/resources/correctness/kibana_sample_data_flights.csv diff --git a/src/test/resources/correctness/kibana_sample_data_flights.json b/integ-test/src/test/resources/correctness/kibana_sample_data_flights.json similarity index 100% rename from src/test/resources/correctness/kibana_sample_data_flights.json rename to integ-test/src/test/resources/correctness/kibana_sample_data_flights.json diff --git a/integ-test/src/test/resources/correctness/queries/groupby.txt b/integ-test/src/test/resources/correctness/queries/groupby.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/integ-test/src/test/resources/correctness/queries/joins.txt b/integ-test/src/test/resources/correctness/queries/joins.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/integ-test/src/test/resources/correctness/queries/select.txt b/integ-test/src/test/resources/correctness/queries/select.txt new file mode 100644 index 0000000000..18d34e7ee9 --- /dev/null +++ b/integ-test/src/test/resources/correctness/queries/select.txt @@ -0,0 +1,2 @@ +SELECT DistanceMiles FROM kibana_sample_data_flights +SELECT AvgTicketPrice, Carrier FROM kibana_sample_data_flights WHERE AvgTicketPrice <= 500 diff --git a/integ-test/src/test/resources/correctness/queries/subquries.txt b/integ-test/src/test/resources/correctness/queries/subquries.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/test/resources/correctness/sanity_integration_tests.txt b/integ-test/src/test/resources/correctness/sanity_integration_tests.txt similarity index 100% rename from src/test/resources/correctness/sanity_integration_tests.txt rename to integ-test/src/test/resources/correctness/sanity_integration_tests.txt diff --git a/src/test/resources/correctness/tableau_integration_tests.txt b/integ-test/src/test/resources/correctness/tableau_integration_tests.txt similarity index 100% rename from src/test/resources/correctness/tableau_integration_tests.txt rename to integ-test/src/test/resources/correctness/tableau_integration_tests.txt diff --git a/src/test/resources/correctness/test_data_test.csv b/integ-test/src/test/resources/correctness/test_data_test.csv similarity index 100% rename from src/test/resources/correctness/test_data_test.csv rename to integ-test/src/test/resources/correctness/test_data_test.csv diff --git a/src/test/resources/correctness/test_data_test.json b/integ-test/src/test/resources/correctness/test_data_test.json similarity index 100% rename from src/test/resources/correctness/test_data_test.json rename to integ-test/src/test/resources/correctness/test_data_test.json diff --git a/src/test/resources/dates.json b/integ-test/src/test/resources/dates.json similarity index 100% rename from src/test/resources/dates.json rename to integ-test/src/test/resources/dates.json diff --git a/src/test/resources/datetime.json b/integ-test/src/test/resources/datetime.json similarity index 100% rename from src/test/resources/datetime.json rename to integ-test/src/test/resources/datetime.json diff --git a/src/test/resources/doctest/mappings/accounts.json b/integ-test/src/test/resources/doctest/mappings/accounts.json similarity index 100% rename from src/test/resources/doctest/mappings/accounts.json rename to integ-test/src/test/resources/doctest/mappings/accounts.json diff --git a/src/test/resources/doctest/mappings/employees_nested.json b/integ-test/src/test/resources/doctest/mappings/employees_nested.json similarity index 100% rename from src/test/resources/doctest/mappings/employees_nested.json rename to integ-test/src/test/resources/doctest/mappings/employees_nested.json diff --git a/src/test/resources/doctest/templates/admin/monitoring.rst b/integ-test/src/test/resources/doctest/templates/admin/monitoring.rst similarity index 100% rename from src/test/resources/doctest/templates/admin/monitoring.rst rename to integ-test/src/test/resources/doctest/templates/admin/monitoring.rst diff --git a/src/test/resources/doctest/templates/admin/settings.rst b/integ-test/src/test/resources/doctest/templates/admin/settings.rst similarity index 100% rename from src/test/resources/doctest/templates/admin/settings.rst rename to integ-test/src/test/resources/doctest/templates/admin/settings.rst diff --git a/src/test/resources/doctest/templates/beyond/fulltext.rst b/integ-test/src/test/resources/doctest/templates/beyond/fulltext.rst similarity index 100% rename from src/test/resources/doctest/templates/beyond/fulltext.rst rename to integ-test/src/test/resources/doctest/templates/beyond/fulltext.rst diff --git a/src/test/resources/doctest/templates/beyond/partiql.rst b/integ-test/src/test/resources/doctest/templates/beyond/partiql.rst similarity index 100% rename from src/test/resources/doctest/templates/beyond/partiql.rst rename to integ-test/src/test/resources/doctest/templates/beyond/partiql.rst diff --git a/src/test/resources/doctest/templates/dml/delete.rst b/integ-test/src/test/resources/doctest/templates/dml/delete.rst similarity index 100% rename from src/test/resources/doctest/templates/dml/delete.rst rename to integ-test/src/test/resources/doctest/templates/dml/delete.rst diff --git a/src/test/resources/doctest/templates/dql/basics.rst b/integ-test/src/test/resources/doctest/templates/dql/basics.rst similarity index 100% rename from src/test/resources/doctest/templates/dql/basics.rst rename to integ-test/src/test/resources/doctest/templates/dql/basics.rst diff --git a/src/test/resources/doctest/templates/dql/complex.rst b/integ-test/src/test/resources/doctest/templates/dql/complex.rst similarity index 100% rename from src/test/resources/doctest/templates/dql/complex.rst rename to integ-test/src/test/resources/doctest/templates/dql/complex.rst diff --git a/src/test/resources/doctest/templates/dql/functions.rst b/integ-test/src/test/resources/doctest/templates/dql/functions.rst similarity index 100% rename from src/test/resources/doctest/templates/dql/functions.rst rename to integ-test/src/test/resources/doctest/templates/dql/functions.rst diff --git a/src/test/resources/doctest/templates/dql/metadata.rst b/integ-test/src/test/resources/doctest/templates/dql/metadata.rst similarity index 100% rename from src/test/resources/doctest/templates/dql/metadata.rst rename to integ-test/src/test/resources/doctest/templates/dql/metadata.rst diff --git a/src/test/resources/doctest/templates/interfaces/endpoint.rst b/integ-test/src/test/resources/doctest/templates/interfaces/endpoint.rst similarity index 100% rename from src/test/resources/doctest/templates/interfaces/endpoint.rst rename to integ-test/src/test/resources/doctest/templates/interfaces/endpoint.rst diff --git a/src/test/resources/doctest/templates/interfaces/protocol.rst b/integ-test/src/test/resources/doctest/templates/interfaces/protocol.rst similarity index 100% rename from src/test/resources/doctest/templates/interfaces/protocol.rst rename to integ-test/src/test/resources/doctest/templates/interfaces/protocol.rst diff --git a/src/test/resources/doctest/testdata/accounts.json b/integ-test/src/test/resources/doctest/testdata/accounts.json similarity index 100% rename from src/test/resources/doctest/testdata/accounts.json rename to integ-test/src/test/resources/doctest/testdata/accounts.json diff --git a/src/test/resources/doctest/testdata/employees_nested.json b/integ-test/src/test/resources/doctest/testdata/employees_nested.json similarity index 100% rename from src/test/resources/doctest/testdata/employees_nested.json rename to integ-test/src/test/resources/doctest/testdata/employees_nested.json diff --git a/src/test/resources/dogs.json b/integ-test/src/test/resources/dogs.json similarity index 100% rename from src/test/resources/dogs.json rename to integ-test/src/test/resources/dogs.json diff --git a/src/test/resources/dogs2.json b/integ-test/src/test/resources/dogs2.json similarity index 100% rename from src/test/resources/dogs2.json rename to integ-test/src/test/resources/dogs2.json diff --git a/src/test/resources/dogs3.json b/integ-test/src/test/resources/dogs3.json similarity index 100% rename from src/test/resources/dogs3.json rename to integ-test/src/test/resources/dogs3.json diff --git a/src/test/resources/dogsubquery.json b/integ-test/src/test/resources/dogsubquery.json similarity index 100% rename from src/test/resources/dogsubquery.json rename to integ-test/src/test/resources/dogsubquery.json diff --git a/src/test/resources/elasticsearch.yml b/integ-test/src/test/resources/elasticsearch.yml similarity index 100% rename from src/test/resources/elasticsearch.yml rename to integ-test/src/test/resources/elasticsearch.yml diff --git a/src/test/resources/employee_nested.json b/integ-test/src/test/resources/employee_nested.json similarity index 100% rename from src/test/resources/employee_nested.json rename to integ-test/src/test/resources/employee_nested.json diff --git a/src/test/resources/expectedOutput/aggregation_query_explain.json b/integ-test/src/test/resources/expectedOutput/aggregation_query_explain.json similarity index 100% rename from src/test/resources/expectedOutput/aggregation_query_explain.json rename to integ-test/src/test/resources/expectedOutput/aggregation_query_explain.json diff --git a/src/test/resources/expectedOutput/between_query.json b/integ-test/src/test/resources/expectedOutput/between_query.json similarity index 100% rename from src/test/resources/expectedOutput/between_query.json rename to integ-test/src/test/resources/expectedOutput/between_query.json diff --git a/src/test/resources/expectedOutput/delete_explain.json b/integ-test/src/test/resources/expectedOutput/delete_explain.json similarity index 100% rename from src/test/resources/expectedOutput/delete_explain.json rename to integ-test/src/test/resources/expectedOutput/delete_explain.json diff --git a/src/test/resources/expectedOutput/explainIT_format_not_pretty.json b/integ-test/src/test/resources/expectedOutput/explainIT_format_not_pretty.json similarity index 100% rename from src/test/resources/expectedOutput/explainIT_format_not_pretty.json rename to integ-test/src/test/resources/expectedOutput/explainIT_format_not_pretty.json diff --git a/src/test/resources/expectedOutput/explainIT_format_pretty.json b/integ-test/src/test/resources/expectedOutput/explainIT_format_pretty.json similarity index 100% rename from src/test/resources/expectedOutput/explainIT_format_pretty.json rename to integ-test/src/test/resources/expectedOutput/explainIT_format_pretty.json diff --git a/src/test/resources/expectedOutput/multi_match_query.json b/integ-test/src/test/resources/expectedOutput/multi_match_query.json similarity index 100% rename from src/test/resources/expectedOutput/multi_match_query.json rename to integ-test/src/test/resources/expectedOutput/multi_match_query.json diff --git a/src/test/resources/expectedOutput/nested_loop_join_explain.json b/integ-test/src/test/resources/expectedOutput/nested_loop_join_explain.json similarity index 100% rename from src/test/resources/expectedOutput/nested_loop_join_explain.json rename to integ-test/src/test/resources/expectedOutput/nested_loop_join_explain.json diff --git a/src/test/resources/expectedOutput/script_value.json b/integ-test/src/test/resources/expectedOutput/script_value.json similarity index 100% rename from src/test/resources/expectedOutput/script_value.json rename to integ-test/src/test/resources/expectedOutput/script_value.json diff --git a/src/test/resources/expectedOutput/search_explain.json b/integ-test/src/test/resources/expectedOutput/search_explain.json similarity index 100% rename from src/test/resources/expectedOutput/search_explain.json rename to integ-test/src/test/resources/expectedOutput/search_explain.json diff --git a/src/test/resources/expectedOutput/search_explain_filter.json b/integ-test/src/test/resources/expectedOutput/search_explain_filter.json similarity index 100% rename from src/test/resources/expectedOutput/search_explain_filter.json rename to integ-test/src/test/resources/expectedOutput/search_explain_filter.json diff --git a/src/test/resources/expectedOutput/search_spatial_explain.json b/integ-test/src/test/resources/expectedOutput/search_spatial_explain.json similarity index 100% rename from src/test/resources/expectedOutput/search_spatial_explain.json rename to integ-test/src/test/resources/expectedOutput/search_spatial_explain.json diff --git a/src/test/resources/expectedOutput/term_join_where b/integ-test/src/test/resources/expectedOutput/term_join_where similarity index 100% rename from src/test/resources/expectedOutput/term_join_where rename to integ-test/src/test/resources/expectedOutput/term_join_where diff --git a/src/test/resources/expectedOutput/term_union_where b/integ-test/src/test/resources/expectedOutput/term_union_where similarity index 100% rename from src/test/resources/expectedOutput/term_union_where rename to integ-test/src/test/resources/expectedOutput/term_union_where diff --git a/src/test/resources/game_of_thrones_complex.json b/integ-test/src/test/resources/game_of_thrones_complex.json similarity index 100% rename from src/test/resources/game_of_thrones_complex.json rename to integ-test/src/test/resources/game_of_thrones_complex.json diff --git a/integ-test/src/test/resources/indexDefinitions/account_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/account_index_mapping.json new file mode 100644 index 0000000000..a2c5ba8577 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/account_index_mapping.json @@ -0,0 +1,44 @@ +{ + "mappings": { + "properties": { + "gender": { + "type": "text", + "fielddata": true + }, + "address": { + "type": "text", + "fielddata": true + }, + "firstname": { + "type": "text", + "fielddata": true, + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "lastname": { + "type": "text", + "fielddata": true, + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "state": { + "type": "text", + "fielddata": true, + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/bank_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/bank_index_mapping.json new file mode 100644 index 0000000000..cd99b83800 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/bank_index_mapping.json @@ -0,0 +1,52 @@ +{ + "mappings": { + "properties": { + "account_number": { + "type": "long" + }, + "address": { + "type": "text" + }, + "age": { + "type": "integer" + }, + "balance": { + "type": "long" + }, + "birthdate": { + "type": "date" + }, + "city": { + "type": "keyword" + }, + "email": { + "type": "text" + }, + "employer": { + "type": "text" + }, + "firstname": { + "type": "text" + }, + "gender": { + "type": "text", + "fielddata": true + }, + "lastname": { + "type": "keyword" + }, + "male": { + "type": "boolean" + }, + "state": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/bank_with_null_values_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/bank_with_null_values_index_mapping.json new file mode 100644 index 0000000000..b154d1c0cc --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/bank_with_null_values_index_mapping.json @@ -0,0 +1,27 @@ +{ + "mappings": { + "properties": { + "account_number": { + "type": "long" + }, + "address": { + "type": "text" + }, + "age": { + "type": "integer" + }, + "balance": { + "type": "long" + }, + "gender": { + "type": "text" + }, + "firstname": { + "type": "text" + }, + "lastname": { + "type": "keyword" + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/date_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/date_index_mapping.json new file mode 100644 index 0000000000..7862c312a4 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/date_index_mapping.json @@ -0,0 +1,10 @@ +{ + "mappings": { + "properties": { + "date_keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/date_time_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/date_time_index_mapping.json new file mode 100644 index 0000000000..6f1d465a82 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/date_time_index_mapping.json @@ -0,0 +1,9 @@ +{ + "mappings": { + "properties": { + "birthday": { + "type": "date" + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/dog2_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/dog2_index_mapping.json new file mode 100644 index 0000000000..3d7b31dc14 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/dog2_index_mapping.json @@ -0,0 +1,13 @@ +{ + "mappings": { + "properties": { + "dog_name": { + "type": "text", + "fielddata": true + }, + "holdersName": { + "type": "keyword" + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/dog3_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/dog3_index_mapping.json new file mode 100644 index 0000000000..b32cfe7a7b --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/dog3_index_mapping.json @@ -0,0 +1,12 @@ +{ + "mappings": { + "properties": { + "holdersName": { + "type": "keyword" + }, + "color": { + "type": "text" + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/dog_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/dog_index_mapping.json new file mode 100644 index 0000000000..57b35687c9 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/dog_index_mapping.json @@ -0,0 +1,10 @@ +{ + "mappings": { + "properties": { + "dog_name": { + "type": "text", + "fielddata": true + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/employee_nested_type_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/employee_nested_type_index_mapping.json new file mode 100644 index 0000000000..d91738a15c --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/employee_nested_type_index_mapping.json @@ -0,0 +1,87 @@ +{ + "mappings": { + "properties": { + "comments": { + "type": "nested", + "properties": { + "date": { + "type": "date" + }, + "likes": { + "type": "long" + }, + "message": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "id": { + "type": "long" + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "projects": { + "type": "nested", + "properties": { + "address": { + "type": "nested", + "properties": { + "city": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "state": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + }, + "fielddata": true + }, + "started_year": { + "type": "long" + } + } + }, + "title": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/game_of_thrones_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/game_of_thrones_index_mapping.json new file mode 100644 index 0000000000..5e7baf4049 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/game_of_thrones_index_mapping.json @@ -0,0 +1,44 @@ +{ + "mappings": { + "properties": { + "nickname": { + "type": "text", + "fielddata": true + }, + "name": { + "properties": { + "firstname": { + "type": "text", + "fielddata": true + }, + "lastname": { + "type": "text", + "fielddata": true + }, + "ofHerName": { + "type": "integer" + }, + "ofHisName": { + "type": "integer" + } + } + }, + "house": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "gender": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/join_type_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/join_type_index_mapping.json new file mode 100644 index 0000000000..5c666de43e --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/join_type_index_mapping.json @@ -0,0 +1,27 @@ +{ + "mappings": { + "properties": { + "join_field": { + "type": "join", + "relations": { + "parentType": "childrenType" + } + }, + "parentTile": { + "index": "true", + "type": "keyword" + }, + "dayOfWeek": { + "type": "long" + }, + "author": { + "index": "true", + "type": "keyword" + }, + "info": { + "index": "true", + "type": "keyword" + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/location_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/location_index_mapping.json new file mode 100644 index 0000000000..f047b52ead --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/location_index_mapping.json @@ -0,0 +1,15 @@ +{ + "mappings": { + "properties": { + "place": { + "type": "geo_shape" + }, + "center": { + "type": "geo_point" + }, + "description": { + "type": "text" + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/nested_simple_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/nested_simple_index_mapping.json new file mode 100644 index 0000000000..2ebc8a50de --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/nested_simple_index_mapping.json @@ -0,0 +1,44 @@ +{ + "mappings": { + "properties": { + "address": { + "type": "nested", + "properties": { + "city": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "state": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "age": { + "type": "long" + }, + "id": { + "type": "long" + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/nested_type_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/nested_type_index_mapping.json new file mode 100644 index 0000000000..4f0ed97433 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/nested_type_index_mapping.json @@ -0,0 +1,47 @@ +{ + "mappings": { + "properties": { + "message": { + "type": "nested", + "properties": { + "info": { + "type": "keyword", + "index": "true" + }, + "author": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + }, + "index": "true" + }, + "dayOfWeek": { + "type": "long" + } + } + }, + "comment": { + "type": "nested", + "properties": { + "data": { + "type": "keyword", + "index": "true" + }, + "likes": { + "type": "long" + } + } + }, + "myNum": { + "type": "long" + }, + "someField": { + "type": "keyword", + "index": "true" + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/odbc_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/odbc_index_mapping.json new file mode 100644 index 0000000000..ec67d64bc9 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/odbc_index_mapping.json @@ -0,0 +1,13 @@ +{ + "mappings": { + "properties": { + "odbc_time": { + "type": "date", + "format": "'{ts' ''yyyy-MM-dd HH:mm:ss.SSS'''}'" + }, + "docCount": { + "type": "text" + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/order_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/order_index_mapping.json new file mode 100644 index 0000000000..069e8682b3 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/order_index_mapping.json @@ -0,0 +1,18 @@ +{ + "mappings": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/people2_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/people2_index_mapping.json new file mode 100644 index 0000000000..89ff46cadc --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/people2_index_mapping.json @@ -0,0 +1,9 @@ +{ + "mappings": { + "properties": { + "firstname": { + "type": "keyword" + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/phrase_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/phrase_index_mapping.json new file mode 100644 index 0000000000..12025015b5 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/phrase_index_mapping.json @@ -0,0 +1,10 @@ +{ + "mappings": { + "properties": { + "phrase": { + "type": "text", + "store": true + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/weblogs_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/weblogs_index_mapping.json new file mode 100644 index 0000000000..05b9784313 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/weblogs_index_mapping.json @@ -0,0 +1,21 @@ +{ + "mappings": { + "properties": { + "host": { + "type": "ip" + }, + "method": { + "type": "text" + }, + "url": { + "type": "text" + }, + "response": { + "type": "text" + }, + "bytes": { + "type": "text" + } + } + } +} diff --git a/src/test/resources/join_objects.json b/integ-test/src/test/resources/join_objects.json similarity index 100% rename from src/test/resources/join_objects.json rename to integ-test/src/test/resources/join_objects.json diff --git a/src/test/resources/locations.json b/integ-test/src/test/resources/locations.json similarity index 100% rename from src/test/resources/locations.json rename to integ-test/src/test/resources/locations.json diff --git a/src/test/resources/locations2.json b/integ-test/src/test/resources/locations2.json similarity index 100% rename from src/test/resources/locations2.json rename to integ-test/src/test/resources/locations2.json diff --git a/src/test/resources/nested_objects.json b/integ-test/src/test/resources/nested_objects.json similarity index 100% rename from src/test/resources/nested_objects.json rename to integ-test/src/test/resources/nested_objects.json diff --git a/src/test/resources/nested_objects_quotes_in_values.json b/integ-test/src/test/resources/nested_objects_quotes_in_values.json similarity index 100% rename from src/test/resources/nested_objects_quotes_in_values.json rename to integ-test/src/test/resources/nested_objects_quotes_in_values.json diff --git a/src/test/resources/nested_simple.json b/integ-test/src/test/resources/nested_simple.json similarity index 100% rename from src/test/resources/nested_simple.json rename to integ-test/src/test/resources/nested_simple.json diff --git a/src/test/resources/odbc-date-formats.json b/integ-test/src/test/resources/odbc-date-formats.json similarity index 100% rename from src/test/resources/odbc-date-formats.json rename to integ-test/src/test/resources/odbc-date-formats.json diff --git a/src/test/resources/online.json b/integ-test/src/test/resources/online.json similarity index 100% rename from src/test/resources/online.json rename to integ-test/src/test/resources/online.json diff --git a/src/test/resources/order.json b/integ-test/src/test/resources/order.json similarity index 100% rename from src/test/resources/order.json rename to integ-test/src/test/resources/order.json diff --git a/src/test/resources/people2.json b/integ-test/src/test/resources/people2.json similarity index 100% rename from src/test/resources/people2.json rename to integ-test/src/test/resources/people2.json diff --git a/src/test/resources/peoples.json b/integ-test/src/test/resources/peoples.json similarity index 100% rename from src/test/resources/peoples.json rename to integ-test/src/test/resources/peoples.json diff --git a/src/test/resources/phrases.json b/integ-test/src/test/resources/phrases.json similarity index 100% rename from src/test/resources/phrases.json rename to integ-test/src/test/resources/phrases.json diff --git a/src/test/resources/systems.json b/integ-test/src/test/resources/systems.json similarity index 100% rename from src/test/resources/systems.json rename to integ-test/src/test/resources/systems.json diff --git a/src/test/resources/weblogs.json b/integ-test/src/test/resources/weblogs.json similarity index 100% rename from src/test/resources/weblogs.json rename to integ-test/src/test/resources/weblogs.json diff --git a/legacy/build.gradle b/legacy/build.gradle new file mode 100644 index 0000000000..192d718cf9 --- /dev/null +++ b/legacy/build.gradle @@ -0,0 +1,90 @@ +plugins { + id 'java' + id 'io.freefair.lombok' + id 'antlr' +} + +version '1.8.0.0' + +repositories { + mavenCentral() +} + +generateGrammarSource { + arguments += ['-visitor', '-package', 'com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser'] + source = sourceSets.main.antlr + outputDirectory = file("build/generated-src/antlr/main/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/parser") +} + +// Remove ANTLR plugin jars as it's specified as 'compile' dependency internally +configurations { + compile { + extendsFrom = extendsFrom.findAll { it != configurations.antlr } + } +} + +// TODO: fix compiler warnings +compileJava.options.warnings = false +compileJava { + options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) + doFirst { + // TODO: do not fail build on warnings, need to fix all compiler warnings + options.compilerArgs.remove('-Werror') + // TODO: need to fix all java doc format + options.compilerArgs.remove('-Xdoclint:all') + } +} + +// TODO: Similarly, need to fix compiling errors in test source code +compileTestJava.options.warnings = false +compileTestJava { + options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) + doFirst { + options.compilerArgs.remove('-Werror') + options.compilerArgs.remove('-Xdoclint:all') + } +} + +// TODO: Need to update integration test to use ElasticSearch test framework +test { + include '**/*Test.class' + exclude 'com/amazon/opendistroforelasticsearch/sql/intgtest/**' + // Gradle runs unit test using a working directory other and project root + // set 'project.projectDir' property to allow unit test classes to access test resources + // in src/test/resources in current module + systemProperty('project.root', project.projectDir.absolutePath) +} + +dependencies { + compile group: 'com.alibaba', name: 'druid', version:'1.0.15' + compile group: 'org.locationtech.spatial4j', name: 'spatial4j', version:'0.7' + compile group: "org.elasticsearch.plugin", name: 'parent-join-client', version: "${es_version}" + compile group: "org.elasticsearch.plugin", name: 'reindex-client', version: "${es_version}" + compile group: 'com.google.guava', name: 'guava', version:'23.0' + compile group: 'org.json', name: 'json', version:'20180813' + compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.10' + compile group: 'org.elasticsearch', name: 'elasticsearch', version: "${es_version}" + compile project(':sql') + compile project(':common') + compile project(':elasticsearch') + + // ANTLR gradle plugin and runtime dependency + antlr "org.antlr:antlr4:4.7.1" + compile "org.antlr:antlr4-runtime:4.7.1" + + //compileOnly group: 'org.locationtech.jts', name: 'jts-core', version:'1.15.0' + // compileOnly group: 'org.elasticsearch', name: 'elasticsearch', version:'6.5.3' + // compileOnly group: 'com.unboundid', name: 'unboundid-ldapsdk', version:'3.2.0' + // compileOnly group: 'org.bouncycastle', name: 'bcprov-jdk15on', version:'1.58' + // compileOnly group: 'log4j', name: 'log4j', version:'1.2.17' + // compileOnly group: 'org.apache.logging.log4j', name: 'log4j-api', version:'2.7' + // compileOnly group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.7' + compileOnly group: 'javax.servlet', name: 'servlet-api', version:'2.5' + + testCompile group: 'org.hamcrest', name: 'hamcrest-core', version:'2.2' + // testCompile group: 'com.alibaba', name: 'fastjson', version:'1.2.56' + testCompile group: 'org.mockito', name: 'mockito-core', version:'2.23.4' + testCompile group: 'junit', name: 'junit', version: '4.12' + testCompile group: "org.elasticsearch.client", name: 'transport', version: "${es_version}" + +} diff --git a/legacy/lombok.config b/legacy/lombok.config new file mode 100644 index 0000000000..6aa51d71ec --- /dev/null +++ b/legacy/lombok.config @@ -0,0 +1,2 @@ +# This file is generated by the 'io.freefair.lombok' Gradle plugin +config.stopBubbling = true diff --git a/src/assembly/zip.xml b/legacy/src/assembly/zip.xml similarity index 100% rename from src/assembly/zip.xml rename to legacy/src/assembly/zip.xml diff --git a/src/main/antlr/OpenDistroSqlLexer.g4 b/legacy/src/main/antlr/OpenDistroSqlLexer.g4 similarity index 100% rename from src/main/antlr/OpenDistroSqlLexer.g4 rename to legacy/src/main/antlr/OpenDistroSqlLexer.g4 diff --git a/src/main/antlr/OpenDistroSqlParser.g4 b/legacy/src/main/antlr/OpenDistroSqlParser.g4 similarity index 100% rename from src/main/antlr/OpenDistroSqlParser.g4 rename to legacy/src/main/antlr/OpenDistroSqlParser.g4 diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/OpenDistroSqlAnalyzer.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/OpenDistroSqlAnalyzer.java similarity index 76% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/OpenDistroSqlAnalyzer.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/OpenDistroSqlAnalyzer.java index ec69bf8974..2e71de2dff 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/OpenDistroSqlAnalyzer.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/OpenDistroSqlAnalyzer.java @@ -13,20 +13,20 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr; - -import com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlLexer; -import com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.scope.SemanticContext; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.visitor.ESMappingLoader; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.visitor.SemanticAnalyzer; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.visitor.TypeChecker; -import com.amazon.opendistroforelasticsearch.sql.antlr.syntax.CaseInsensitiveCharStream; -import com.amazon.opendistroforelasticsearch.sql.antlr.syntax.SyntaxAnalysisErrorListener; -import com.amazon.opendistroforelasticsearch.sql.antlr.visitor.AntlrSqlParseTreeVisitor; -import com.amazon.opendistroforelasticsearch.sql.antlr.visitor.EarlyExitAnalysisException; -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr; + +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlLexer; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.scope.SemanticContext; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.visitor.ESMappingLoader; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.visitor.SemanticAnalyzer; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.visitor.TypeChecker; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.syntax.CaseInsensitiveCharStream; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.syntax.SyntaxAnalysisErrorListener; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.visitor.AntlrSqlParseTreeVisitor; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.visitor.EarlyExitAnalysisException; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.Lexer; import org.antlr.v4.runtime.tree.ParseTree; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/SimilarSymbols.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/SimilarSymbols.java similarity index 97% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/SimilarSymbols.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/SimilarSymbols.java index 4efd55c707..27a71f08eb 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/SimilarSymbols.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/SimilarSymbols.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr; import org.apache.lucene.search.spell.LevenshteinDistance; import org.apache.lucene.search.spell.StringDistance; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/SqlAnalysisConfig.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/SqlAnalysisConfig.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/SqlAnalysisConfig.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/SqlAnalysisConfig.java index 7981006d0f..b3788ab223 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/SqlAnalysisConfig.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/SqlAnalysisConfig.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr; /** * Configuration for SQL analysis. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/SqlAnalysisException.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/SqlAnalysisException.java similarity index 92% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/SqlAnalysisException.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/SqlAnalysisException.java index c137f53783..2ffb416ca8 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/SqlAnalysisException.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/SqlAnalysisException.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr; /** * SQL query analysis abstract exception. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalysisException.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalysisException.java similarity index 83% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalysisException.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalysisException.java index 3ef29d5e5b..4c53a048f8 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalysisException.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalysisException.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic; -import com.amazon.opendistroforelasticsearch.sql.antlr.SqlAnalysisException; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.SqlAnalysisException; /** * Exception for semantic analysis diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/Environment.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/Environment.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/Environment.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/Environment.java index 533d7e6b3d..9bcb1ff76a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/Environment.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/Environment.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.scope; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.scope; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; import java.util.HashMap; import java.util.Map; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/Namespace.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/Namespace.java similarity index 92% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/Namespace.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/Namespace.java index 014ddc46dd..5a1bbf715e 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/Namespace.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/Namespace.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.scope; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.scope; /** * Namespace of symbol to avoid naming conflict diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/SemanticContext.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/SemanticContext.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/SemanticContext.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/SemanticContext.java index 8ed28cd563..9a6dfddecf 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/SemanticContext.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/SemanticContext.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.scope; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.scope; import java.util.Objects; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/Symbol.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/Symbol.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/Symbol.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/Symbol.java index 17486215be..d648ff1785 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/Symbol.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/Symbol.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.scope; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.scope; /** * Symbol in the scope diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/SymbolTable.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/SymbolTable.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/SymbolTable.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/SymbolTable.java index ed7907718c..8e4a9599bb 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/SymbolTable.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/SymbolTable.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.scope; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.scope; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; import java.util.EnumMap; import java.util.Map; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/TypeSupplier.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/TypeSupplier.java similarity index 88% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/TypeSupplier.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/TypeSupplier.java index e1605c6f4c..b8f50770f0 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/TypeSupplier.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/TypeSupplier.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.scope; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.scope; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.SemanticAnalysisException; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.SemanticAnalysisException; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; import java.util.HashSet; import java.util.Set; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/Type.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/Type.java similarity index 82% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/Type.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/Type.java index 28a73d141e..045d82091b 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/Type.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/Type.java @@ -13,17 +13,17 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.SemanticAnalysisException; -import com.amazon.opendistroforelasticsearch.sql.antlr.visitor.Reducible; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.SemanticAnalysisException; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.visitor.Reducible; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; import java.util.List; import java.util.stream.Collectors; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.TYPE_ERROR; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.UNKNOWN; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.TYPE_ERROR; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.UNKNOWN; /** * Type interface which represents any type of symbol in the SQL. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/TypeExpression.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/TypeExpression.java similarity index 91% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/TypeExpression.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/TypeExpression.java index f80d1f6945..263d390661 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/TypeExpression.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/TypeExpression.java @@ -13,18 +13,18 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.special.Generic; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.special.Generic; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; import java.util.Arrays; import java.util.List; import java.util.function.Function; import java.util.stream.Collectors; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.TYPE_ERROR; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.UNKNOWN; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.TYPE_ERROR; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.UNKNOWN; /** * Type expression representing specification(s) of constructor such as function, operator etc. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/base/BaseType.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/base/BaseType.java similarity index 84% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/base/BaseType.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/base/BaseType.java index 80d59186dc..49edc8862f 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/base/BaseType.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/base/BaseType.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/base/ESDataType.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/base/ESDataType.java similarity index 89% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/base/ESDataType.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/base/ESDataType.java index 1be5440866..8550d02d94 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/base/ESDataType.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/base/ESDataType.java @@ -13,15 +13,15 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; import com.google.common.collect.ImmutableMap; import java.util.Map; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESIndex.IndexType.NESTED_FIELD; -import static com.amazon.opendistroforelasticsearch.sql.utils.StringUtils.toUpper; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESIndex.IndexType.NESTED_FIELD; +import static com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils.toUpper; /** * Base type hierarchy based on Elasticsearch data type diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/base/ESIndex.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/base/ESIndex.java similarity index 92% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/base/ESIndex.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/base/ESIndex.java index b9b42654f1..11c23077be 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/base/ESIndex.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/base/ESIndex.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; import java.util.Objects; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/function/AggregateFunction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/function/AggregateFunction.java similarity index 65% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/function/AggregateFunction.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/function/AggregateFunction.java index 8b1f642f0b..c1c806723b 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/function/AggregateFunction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/function/AggregateFunction.java @@ -13,16 +13,16 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.function; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.function; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.TypeExpression; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.TypeExpression; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.DOUBLE; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.ES_TYPE; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.INTEGER; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.NUMBER; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.special.Generic.T; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.ES_TYPE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.NUMBER; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.special.Generic.T; /** * Aggregate function diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/function/ESScalarFunction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/function/ESScalarFunction.java similarity index 78% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/function/ESScalarFunction.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/function/ESScalarFunction.java index 4b8f2a9e45..d142229493 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/function/ESScalarFunction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/function/ESScalarFunction.java @@ -13,17 +13,17 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.function; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.function; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.TypeExpression; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.TypeExpression; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.BOOLEAN; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.DATE; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.GEO_POINT; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.INTEGER; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.NUMBER; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.BOOLEAN; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.DATE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.GEO_POINT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.NUMBER; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.STRING; /** * Elasticsearch special scalar functions diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/function/ScalarFunction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/function/ScalarFunction.java similarity index 79% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/function/ScalarFunction.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/function/ScalarFunction.java index 4602f3ce0d..7ba1e0f618 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/function/ScalarFunction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/function/ScalarFunction.java @@ -13,19 +13,19 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.function; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.function; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.TypeExpression; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.TypeExpression; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.BOOLEAN; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.DOUBLE; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.ES_TYPE; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.INTEGER; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.NUMBER; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.STRING; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.special.Generic.T; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.BOOLEAN; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.ES_TYPE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.NUMBER; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.special.Generic.T; /** * Scalar SQL function diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/operator/ComparisonOperator.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/operator/ComparisonOperator.java similarity index 81% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/operator/ComparisonOperator.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/operator/ComparisonOperator.java index 90b96f5cc1..15b9cacc52 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/operator/ComparisonOperator.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/operator/ComparisonOperator.java @@ -13,14 +13,14 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.operator; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.operator; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; import java.util.List; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.BOOLEAN; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.TYPE_ERROR; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.BOOLEAN; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.TYPE_ERROR; /** * Type for comparison operator diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/operator/JoinOperator.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/operator/JoinOperator.java similarity index 79% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/operator/JoinOperator.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/operator/JoinOperator.java index ffba99ca60..02b06744b5 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/operator/JoinOperator.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/operator/JoinOperator.java @@ -13,15 +13,15 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.operator; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.operator; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESIndex; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESIndex; import java.util.List; import java.util.Optional; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.TYPE_ERROR; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.TYPE_ERROR; /** * Join operator diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/operator/SetOperator.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/operator/SetOperator.java similarity index 85% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/operator/SetOperator.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/operator/SetOperator.java index 5503c31b16..7920f19cdf 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/operator/SetOperator.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/operator/SetOperator.java @@ -13,13 +13,13 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.operator; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.operator; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; import java.util.List; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.TYPE_ERROR; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.TYPE_ERROR; /** * Set operator between queries. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/special/Generic.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/special/Generic.java similarity index 92% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/special/Generic.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/special/Generic.java index 64f6025ad3..1873b1f6a3 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/special/Generic.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/special/Generic.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.special; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.special; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; import java.util.Arrays; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/special/Product.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/special/Product.java similarity index 92% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/special/Product.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/special/Product.java index a07eaabfab..2e6e75081d 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/special/Product.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/special/Product.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.special; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.special; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; import lombok.Getter; import java.util.Collections; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/ESMappingLoader.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/visitor/ESMappingLoader.java similarity index 83% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/ESMappingLoader.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/visitor/ESMappingLoader.java index 4ac080006b..d353999691 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/ESMappingLoader.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/visitor/ESMappingLoader.java @@ -13,28 +13,28 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.visitor; - -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.scope.Environment; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.scope.Namespace; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.scope.SemanticContext; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.scope.Symbol; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESIndex; -import com.amazon.opendistroforelasticsearch.sql.antlr.visitor.EarlyExitAnalysisException; -import com.amazon.opendistroforelasticsearch.sql.antlr.visitor.GenericSqlParseTreeVisitor; -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; -import com.amazon.opendistroforelasticsearch.sql.esdomain.mapping.FieldMappings; -import com.amazon.opendistroforelasticsearch.sql.esdomain.mapping.IndexMappings; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.visitor; + +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.scope.Environment; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.scope.Namespace; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.scope.SemanticContext; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.scope.Symbol; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESIndex; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.visitor.EarlyExitAnalysisException; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.visitor.GenericSqlParseTreeVisitor; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.mapping.FieldMappings; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.mapping.IndexMappings; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESIndex.IndexType.INDEX; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESIndex.IndexType.NESTED_FIELD; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESIndex.IndexType.INDEX; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESIndex.IndexType.NESTED_FIELD; /** * Load index and nested field mapping into semantic context diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/SemanticAnalyzer.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/visitor/SemanticAnalyzer.java similarity index 89% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/SemanticAnalyzer.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/visitor/SemanticAnalyzer.java index b40decbc85..a795772855 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/SemanticAnalyzer.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/visitor/SemanticAnalyzer.java @@ -13,15 +13,15 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.visitor; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.visitor; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; -import com.amazon.opendistroforelasticsearch.sql.antlr.visitor.GenericSqlParseTreeVisitor; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.visitor.GenericSqlParseTreeVisitor; import java.util.List; -import static com.amazon.opendistroforelasticsearch.sql.utils.StringUtils.unquoteFullColumn; -import static com.amazon.opendistroforelasticsearch.sql.utils.StringUtils.unquoteSingleField; +import static com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils.unquoteFullColumn; +import static com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils.unquoteSingleField; /** * Main visitor implementation to drive the entire semantic analysis. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/TypeChecker.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/visitor/TypeChecker.java similarity index 77% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/TypeChecker.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/visitor/TypeChecker.java index 9ac58a24b1..6e2f55bf71 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/visitor/TypeChecker.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/visitor/TypeChecker.java @@ -13,33 +13,33 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.visitor; - -import com.amazon.opendistroforelasticsearch.sql.antlr.SimilarSymbols; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.SemanticAnalysisException; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.scope.Environment; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.scope.Namespace; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.scope.SemanticContext; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.scope.Symbol; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.TypeExpression; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.function.AggregateFunction; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.function.ESScalarFunction; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.function.ScalarFunction; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.operator.ComparisonOperator; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.operator.JoinOperator; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.operator.SetOperator; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.special.Product; -import com.amazon.opendistroforelasticsearch.sql.antlr.visitor.GenericSqlParseTreeVisitor; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.visitor; + +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.SimilarSymbols; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.SemanticAnalysisException; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.scope.Environment; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.scope.Namespace; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.scope.SemanticContext; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.scope.Symbol; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.TypeExpression; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.function.AggregateFunction; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.function.ESScalarFunction; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.function.ScalarFunction; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.operator.ComparisonOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.operator.JoinOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.operator.SetOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.special.Product; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.visitor.GenericSqlParseTreeVisitor; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; import com.google.common.collect.ImmutableList; import java.util.List; import java.util.Optional; import java.util.Set; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.UNKNOWN; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.UNKNOWN; /** * SQL semantic analyzer that determines if a syntactical correct query is meaningful. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/syntax/CaseInsensitiveCharStream.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/syntax/CaseInsensitiveCharStream.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/syntax/CaseInsensitiveCharStream.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/syntax/CaseInsensitiveCharStream.java index 7047cb9d45..febc701168 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/syntax/CaseInsensitiveCharStream.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/syntax/CaseInsensitiveCharStream.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.syntax; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.syntax; import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.CharStreams; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/syntax/SyntaxAnalysisErrorListener.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/syntax/SyntaxAnalysisErrorListener.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/syntax/SyntaxAnalysisErrorListener.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/syntax/SyntaxAnalysisErrorListener.java index 2011072f46..b5dec44e6d 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/syntax/SyntaxAnalysisErrorListener.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/syntax/SyntaxAnalysisErrorListener.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.syntax; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.syntax; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; import org.antlr.v4.runtime.BaseErrorListener; import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.RecognitionException; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/syntax/SyntaxAnalysisException.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/syntax/SyntaxAnalysisException.java similarity index 83% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/syntax/SyntaxAnalysisException.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/syntax/SyntaxAnalysisException.java index 3fa056b25b..abfc2766f4 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/syntax/SyntaxAnalysisException.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/syntax/SyntaxAnalysisException.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.syntax; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.syntax; -import com.amazon.opendistroforelasticsearch.sql.antlr.SqlAnalysisException; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.SqlAnalysisException; /** * Exception for syntax analysis diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/visitor/AntlrSqlParseTreeVisitor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/visitor/AntlrSqlParseTreeVisitor.java similarity index 73% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/visitor/AntlrSqlParseTreeVisitor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/visitor/AntlrSqlParseTreeVisitor.java index 6cfd665e93..b6ce33d025 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/visitor/AntlrSqlParseTreeVisitor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/visitor/AntlrSqlParseTreeVisitor.java @@ -13,15 +13,15 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.visitor; - -import com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser; -import com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.InnerJoinContext; -import com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.QuerySpecificationContext; -import com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.SelectColumnElementContext; -import com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.SubqueryTableItemContext; -import com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.TableNamePatternContext; -import com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParserBaseVisitor; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.visitor; + +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.InnerJoinContext; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.QuerySpecificationContext; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.SelectColumnElementContext; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.SubqueryTableItemContext; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.TableNamePatternContext; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParserBaseVisitor; import com.google.common.base.Strings; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.tree.ParseTree; @@ -31,37 +31,37 @@ import java.util.List; import java.util.stream.Collectors; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.AggregateWindowedFunctionContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.AtomTableItemContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.BinaryComparisonPredicateContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.BooleanLiteralContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.ComparisonOperatorContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.ConstantContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.DecimalLiteralContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.FromClauseContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.FullColumnNameContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.FunctionNameBaseContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.InPredicateContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.IsExpressionContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.MathOperatorContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.MinusSelectContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.OuterJoinContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.PredicateContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.RegexpPredicateContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.RootContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.ScalarFunctionCallContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.SelectElementsContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.SelectExpressionElementContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.SelectFunctionElementContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.SimpleTableNameContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.StringLiteralContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.TableAndTypeNameContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.TableSourceBaseContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.TableSourceItemContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.TableSourcesContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.UdfFunctionCallContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.UidContext; -import static com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.UnionSelectContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.AggregateWindowedFunctionContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.AtomTableItemContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.BinaryComparisonPredicateContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.BooleanLiteralContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.ComparisonOperatorContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.ConstantContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.DecimalLiteralContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.FromClauseContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.FullColumnNameContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.FunctionNameBaseContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.InPredicateContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.IsExpressionContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.MathOperatorContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.MinusSelectContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.OuterJoinContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.PredicateContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.RegexpPredicateContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.RootContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.ScalarFunctionCallContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.SelectElementsContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.SelectExpressionElementContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.SelectFunctionElementContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.SimpleTableNameContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.StringLiteralContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.TableAndTypeNameContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.TableSourceBaseContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.TableSourceItemContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.TableSourcesContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.UdfFunctionCallContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.UidContext; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.UnionSelectContext; import static java.util.Collections.emptyList; import static java.util.Collections.singleton; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/visitor/EarlyExitAnalysisException.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/visitor/EarlyExitAnalysisException.java similarity index 91% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/visitor/EarlyExitAnalysisException.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/visitor/EarlyExitAnalysisException.java index c9ee1d0289..96e8b80bf8 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/visitor/EarlyExitAnalysisException.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/visitor/EarlyExitAnalysisException.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.visitor; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.visitor; /** * Exit visitor early due to some reason. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/visitor/GenericSqlParseTreeVisitor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/visitor/GenericSqlParseTreeVisitor.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/visitor/GenericSqlParseTreeVisitor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/visitor/GenericSqlParseTreeVisitor.java index 553ba2fe84..1da23a19b5 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/visitor/GenericSqlParseTreeVisitor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/visitor/GenericSqlParseTreeVisitor.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.visitor; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.visitor; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/visitor/Reducible.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/visitor/Reducible.java similarity index 92% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/visitor/Reducible.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/visitor/Reducible.java index 7fd986cd45..e8866fa597 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/visitor/Reducible.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/visitor/Reducible.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.visitor; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.visitor; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/visitor/UnsupportedSemanticVerifier.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/visitor/UnsupportedSemanticVerifier.java similarity index 85% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/visitor/UnsupportedSemanticVerifier.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/visitor/UnsupportedSemanticVerifier.java index d4068f8605..c48ad88dd4 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/visitor/UnsupportedSemanticVerifier.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/visitor/UnsupportedSemanticVerifier.java @@ -13,18 +13,18 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.visitor; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.visitor; -import com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlFeatureNotImplementedException; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlFeatureNotImplementedException; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; import com.google.common.collect.Sets; import java.util.Set; -import com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.MathOperatorContext; -import com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.RegexpPredicateContext; -import com.amazon.opendistroforelasticsearch.sql.antlr.parser.OpenDistroSqlParser.ScalarFunctionCallContext; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.MathOperatorContext; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.RegexpPredicateContext; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.parser.OpenDistroSqlParser.ScalarFunctionCallContext; public class UnsupportedSemanticVerifier { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/cursor/Cursor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/cursor/Cursor.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/cursor/Cursor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/cursor/Cursor.java index 54f738bb21..057cac1dd6 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/cursor/Cursor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/cursor/Cursor.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.cursor; +package com.amazon.opendistroforelasticsearch.sql.legacy.cursor; public interface Cursor { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/cursor/CursorType.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/cursor/CursorType.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/cursor/CursorType.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/cursor/CursorType.java index 13a3d86e9f..f48437bdaa 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/cursor/CursorType.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/cursor/CursorType.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.cursor; +package com.amazon.opendistroforelasticsearch.sql.legacy.cursor; import java.util.HashMap; import java.util.Map; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/cursor/DefaultCursor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/cursor/DefaultCursor.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/cursor/DefaultCursor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/cursor/DefaultCursor.java index 1f97ec6369..4bfd137a9a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/cursor/DefaultCursor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/cursor/DefaultCursor.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.cursor; +package com.amazon.opendistroforelasticsearch.sql.legacy.cursor; -import com.amazon.opendistroforelasticsearch.sql.executor.format.Schema; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.Schema; import com.google.common.base.Strings; import lombok.Getter; import lombok.NoArgsConstructor; @@ -52,7 +52,7 @@ public class DefaultCursor implements Cursor { private static final String FIELD_ALIAS_MAP = "a"; /** To get mappings for index to check if type is date needed for - * @see com.amazon.opendistroforelasticsearch.sql.executor.format.DateFieldFormatter */ + * @see com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.DateFieldFormatter */ @NonNull private String indexPattern; @@ -69,7 +69,7 @@ public class DefaultCursor implements Cursor { */ private long rowsLeft; - /** @see com.amazon.opendistroforelasticsearch.sql.executor.format.SelectResultSet */ + /** @see com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.SelectResultSet */ @NonNull private Map fieldAliasMap; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/cursor/NullCursor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/cursor/NullCursor.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/cursor/NullCursor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/cursor/NullCursor.java index cc74b5b191..c876d8dfc5 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/cursor/NullCursor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/cursor/NullCursor.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.cursor; +package com.amazon.opendistroforelasticsearch.sql.legacy.cursor; /** * A placeholder Cursor implementation to work with non-paginated queries. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/ColumnTypeProvider.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/ColumnTypeProvider.java similarity index 88% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/ColumnTypeProvider.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/ColumnTypeProvider.java index 3640cf6da9..a12e33b52a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/ColumnTypeProvider.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/ColumnTypeProvider.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.special.Product; -import com.amazon.opendistroforelasticsearch.sql.executor.format.Schema; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.special.Product; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.Schema; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Condition.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Condition.java similarity index 97% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Condition.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Condition.java index c450364d84..25ac0bc210 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Condition.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Condition.java @@ -13,13 +13,13 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain; import com.alibaba.druid.sql.ast.SQLExpr; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.parser.ChildrenType; -import com.amazon.opendistroforelasticsearch.sql.parser.NestedType; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.ChildrenType; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.NestedType; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Delete.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Delete.java similarity index 91% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Delete.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Delete.java index d291531c9a..622692b879 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Delete.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Delete.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain; /** * SQL Delete statement. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Field.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Field.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Field.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Field.java index 3ca8cca3a9..28390d01a3 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Field.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Field.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLAggregateOption; -import com.amazon.opendistroforelasticsearch.sql.parser.ChildrenType; -import com.amazon.opendistroforelasticsearch.sql.parser.NestedType; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.ChildrenType; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.NestedType; import java.util.Objects; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/From.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/From.java similarity index 97% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/From.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/From.java index 6f65c4211b..6720408eaf 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/From.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/From.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain; /** diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Having.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Having.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Having.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Having.java index e8ff690a54..3635f360f8 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Having.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Having.java @@ -13,15 +13,15 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.statement.SQLSelectGroupByClause; import com.alibaba.druid.util.StringUtils; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.parser.HavingParser; -import com.amazon.opendistroforelasticsearch.sql.parser.NestedType; -import com.amazon.opendistroforelasticsearch.sql.parser.WhereParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.HavingParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.NestedType; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.WhereParser; import com.google.common.collect.Iterables; import org.elasticsearch.script.Script; import org.elasticsearch.search.aggregations.AggregationBuilder; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/IndexStatement.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/IndexStatement.java similarity index 97% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/IndexStatement.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/IndexStatement.java index 9609793b9c..4534deb6f3 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/IndexStatement.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/IndexStatement.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain; /** * Class used to differentiate SHOW and DESCRIBE statements diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/JoinSelect.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/JoinSelect.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/JoinSelect.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/JoinSelect.java index 0521ad65e8..03cae62cf5 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/JoinSelect.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/JoinSelect.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain; import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource; -import com.amazon.opendistroforelasticsearch.sql.domain.hints.Hint; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints.Hint; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/KVValue.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/KVValue.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/KVValue.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/KVValue.java index 0bf007aca4..44c8fb27a5 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/KVValue.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/KVValue.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain; public class KVValue implements Cloneable { public String key; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/MethodField.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/MethodField.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/MethodField.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/MethodField.java index f41f67ef5d..9a4a586854 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/MethodField.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/MethodField.java @@ -13,11 +13,11 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain; import com.alibaba.druid.sql.ast.expr.SQLAggregateOption; -import com.amazon.opendistroforelasticsearch.sql.parser.NestedType; -import com.amazon.opendistroforelasticsearch.sql.utils.Util; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.NestedType; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.Util; import java.util.HashMap; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Order.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Order.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Order.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Order.java index 88ab8b9ae0..84f26d5011 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Order.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Order.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain; /** * 排序规则 diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Paramer.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Paramer.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Paramer.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Paramer.java index 933bb139f9..7f30e4f071 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Paramer.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Paramer.java @@ -13,15 +13,15 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; import com.alibaba.druid.sql.ast.expr.SQLCharExpr; import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr; import com.alibaba.druid.sql.ast.expr.SQLNumericLiteralExpr; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.utils.Util; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.Util; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.index.query.MatchPhraseQueryBuilder; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Query.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Query.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Query.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Query.java index 424a39906a..f3350704fa 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Query.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Query.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain; import java.util.ArrayList; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/QueryActionRequest.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/QueryActionRequest.java similarity index 86% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/QueryActionRequest.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/QueryActionRequest.java index ed0e478eeb..767044d6ce 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/QueryActionRequest.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/QueryActionRequest.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain; -import com.amazon.opendistroforelasticsearch.sql.executor.Format; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.Format; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/QueryStatement.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/QueryStatement.java similarity index 91% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/QueryStatement.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/QueryStatement.java index 06d9feb9a4..85aee6fbd3 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/QueryStatement.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/QueryStatement.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain; /** * Identifier interface used to encompass Query and IndexStatements diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/ScriptMethodField.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/ScriptMethodField.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/ScriptMethodField.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/ScriptMethodField.java index 43720b1071..578fe6c5fe 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/ScriptMethodField.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/ScriptMethodField.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain; import com.alibaba.druid.sql.ast.expr.SQLAggregateOption; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/SearchResult.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/SearchResult.java similarity index 97% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/SearchResult.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/SearchResult.java index 2222fac52d..e494dbcf36 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/SearchResult.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/SearchResult.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.search.SearchHit; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Select.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Select.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Select.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Select.java index a9b9a4b6c6..43ac7f8aa0 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Select.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Select.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain; -import com.amazon.opendistroforelasticsearch.sql.domain.hints.Hint; -import com.amazon.opendistroforelasticsearch.sql.parser.SubQueryExpression; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints.Hint; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.SubQueryExpression; import com.google.common.collect.ImmutableSet; import java.util.ArrayList; @@ -24,7 +24,7 @@ import java.util.Set; import static com.alibaba.druid.sql.ast.statement.SQLJoinTableSource.JoinType; -import static com.amazon.opendistroforelasticsearch.sql.domain.Field.STAR; +import static com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field.STAR; /** diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/TableOnJoinSelect.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/TableOnJoinSelect.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/TableOnJoinSelect.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/TableOnJoinSelect.java index ca12b7d502..30d09015b0 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/TableOnJoinSelect.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/TableOnJoinSelect.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Where.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Where.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Where.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Where.java index b24e2fa9dd..8b7f436f76 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Where.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/Where.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain; import java.util.LinkedList; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/bucketpath/BucketPath.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/bucketpath/BucketPath.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/bucketpath/BucketPath.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/bucketpath/BucketPath.java index 0fb382d13d..9c405ac3e1 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/bucketpath/BucketPath.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/bucketpath/BucketPath.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain.bucketpath; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain.bucketpath; import java.util.ArrayDeque; import java.util.Deque; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/bucketpath/Path.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/bucketpath/Path.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/bucketpath/Path.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/bucketpath/Path.java index e1eb2e2d4e..52ba6eea30 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/bucketpath/Path.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/bucketpath/Path.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain.bucketpath; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain.bucketpath; public class Path { private final String path; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/hints/Hint.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/hints/Hint.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/hints/Hint.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/hints/Hint.java index 66ad8209e1..99bc087400 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/hints/Hint.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/hints/Hint.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain.hints; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints; /** * Created by Eliran on 5/9/2015. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/hints/HintFactory.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/hints/HintFactory.java similarity index 98% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/hints/HintFactory.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/hints/HintFactory.java index aa7331b749..b6a17587cc 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/hints/HintFactory.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/hints/HintFactory.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain.hints; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.dataformat.yaml.YAMLParser; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/hints/HintType.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/hints/HintType.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/hints/HintType.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/hints/HintType.java index c1cd617375..51125df250 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/hints/HintType.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/domain/hints/HintType.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.domain.hints; +package com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints; /** * Created by Eliran on 29/8/2015. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/ESClient.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/ESClient.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/ESClient.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/ESClient.java index 39744f9dba..464e521ee2 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/ESClient.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/ESClient.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.esdomain; +package com.amazon.opendistroforelasticsearch.sql.legacy.esdomain; -import com.amazon.opendistroforelasticsearch.sql.query.join.BackOffRetryStrategy; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.BackOffRetryStrategy; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.search.MultiSearchRequest; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/LocalClusterState.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/LocalClusterState.java index 2d33ea53e2..b9fbf1d589 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/LocalClusterState.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.esdomain; +package com.amazon.opendistroforelasticsearch.sql.legacy.esdomain; -import com.amazon.opendistroforelasticsearch.sql.esdomain.mapping.IndexMappings; -import com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.mapping.IndexMappings; +import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import org.apache.logging.log4j.LogManager; @@ -109,11 +109,11 @@ public void setClusterService(ClusterService clusterService) { this.clusterService = clusterService; clusterService.addListener(event -> { - if (event.metaDataChanged()) { + if (event.metadataChanged()) { // State in cluster service is already changed to event.state() before listener fired if (LOG.isDebugEnabled()) { LOG.debug("Metadata in cluster state changed: {}", - new IndexMappings(clusterService.state().metaData())); + new IndexMappings(clusterService.state().metadata())); } cache.invalidateAll(); } @@ -222,7 +222,7 @@ private IndexMappings findMappings(ClusterState state, String[] indices, String[ Function> fieldFilter) throws IOException { LOG.debug("Cache didn't help. Load and parse mapping in cluster state"); return new IndexMappings( - state.metaData().findMappings(indices, types, fieldFilter) + state.metadata().findMappings(indices, types, fieldFilter) ); } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMapping.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/mapping/FieldMapping.java similarity index 90% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMapping.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/mapping/FieldMapping.java index 0bbacc0327..a586fc60fd 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMapping.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/mapping/FieldMapping.java @@ -13,15 +13,15 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.esdomain.mapping; +package com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.mapping; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; import java.util.Map; import static java.util.Collections.emptyMap; -import static org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData; +import static org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetadata; /** * Field mapping that parses native ES mapping. @@ -39,7 +39,7 @@ public class FieldMapping { /** * Native mapping information returned from ES */ - private final Map typeMappings; + private final Map typeMappings; /** * Maps a field name to Field object that specified in query explicitly @@ -51,7 +51,7 @@ public FieldMapping(String fieldName) { } public FieldMapping(String fieldName, - Map typeMappings, + Map typeMappings, Map specifiedFieldByNames) { this.fieldName = fieldName; @@ -123,7 +123,7 @@ public String path() { */ @SuppressWarnings("unchecked") public String type() { - FieldMappingMetaData metaData = typeMappings.get(fieldName); + FieldMappingMetadata metaData = typeMappings.get(fieldName); Map source = metaData.sourceAsMap(); String[] fieldPath = fieldName.split("\\."); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMappings.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/mapping/FieldMappings.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMappings.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/mapping/FieldMappings.java index f21a320d4c..e2fa4dceb9 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMappings.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/mapping/FieldMappings.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.esdomain.mapping; +package com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.mapping; -import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.cluster.metadata.MappingMetadata; import org.json.JSONObject; import java.util.HashMap; @@ -61,7 +61,7 @@ public class FieldMappings implements Mappings> { */ private final Map fieldMappings; - public FieldMappings(MappingMetaData mappings) { + public FieldMappings(MappingMetadata mappings) { fieldMappings = mappings.sourceAsMap(); } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/IndexMappings.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/mapping/IndexMappings.java similarity index 90% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/IndexMappings.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/mapping/IndexMappings.java index 283ba3ff30..6cf0328baf 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/IndexMappings.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/mapping/IndexMappings.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.esdomain.mapping; +package com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.mapping; -import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.collect.ImmutableOpenMap; import java.util.Map; @@ -55,12 +55,12 @@ public IndexMappings() { this.indexMappings = emptyMap(); } - public IndexMappings(MetaData metaData) { + public IndexMappings(Metadata metaData) { this.indexMappings = buildMappings(metaData.indices(), indexMetaData -> new TypeMappings(indexMetaData.getMappings())); } - public IndexMappings(ImmutableOpenMap> mappings) { + public IndexMappings(ImmutableOpenMap> mappings) { this.indexMappings = buildMappings(mappings, TypeMappings::new); } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/Mappings.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/mapping/Mappings.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/Mappings.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/mapping/Mappings.java index 156832e47f..b89e8b8c3a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/Mappings.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/mapping/Mappings.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.esdomain.mapping; +package com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.mapping; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import com.google.common.collect.ImmutableMap; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/TypeMappings.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/mapping/TypeMappings.java similarity index 89% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/TypeMappings.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/mapping/TypeMappings.java index 00f7b849c0..e3cda1b056 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/TypeMappings.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/mapping/TypeMappings.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.esdomain.mapping; +package com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.mapping; -import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.collect.ImmutableOpenMap; import java.util.Map; @@ -36,7 +36,7 @@ public class TypeMappings implements Mappings { */ private final Map typeMappings; - public TypeMappings(ImmutableOpenMap mappings) { + public TypeMappings(ImmutableOpenMap mappings) { typeMappings = buildMappings(mappings, FieldMappings::new); } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/exception/SQLFeatureDisabledException.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/exception/SQLFeatureDisabledException.java similarity index 92% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/exception/SQLFeatureDisabledException.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/exception/SQLFeatureDisabledException.java index 1109b7fdb4..69578f1d10 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/exception/SQLFeatureDisabledException.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/exception/SQLFeatureDisabledException.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.exception; +package com.amazon.opendistroforelasticsearch.sql.legacy.exception; public class SQLFeatureDisabledException extends Exception { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/exception/SqlFeatureNotImplementedException.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/exception/SqlFeatureNotImplementedException.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/exception/SqlFeatureNotImplementedException.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/exception/SqlFeatureNotImplementedException.java index f40725e345..6da7bb4fd2 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/exception/SqlFeatureNotImplementedException.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/exception/SqlFeatureNotImplementedException.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.exception; +package com.amazon.opendistroforelasticsearch.sql.legacy.exception; /** * Intended for cases when we knowingly omitted some case, letting users know that we didn't implemented feature, but diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/exception/SqlParseException.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/exception/SqlParseException.java similarity index 92% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/exception/SqlParseException.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/exception/SqlParseException.java index 11d4deffd7..0e3754006a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/exception/SqlParseException.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/exception/SqlParseException.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.exception; +package com.amazon.opendistroforelasticsearch.sql.legacy.exception; public class SqlParseException extends Exception { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/ActionRequestRestExecutorFactory.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/ActionRequestRestExecutorFactory.java similarity index 78% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/ActionRequestRestExecutorFactory.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/ActionRequestRestExecutorFactory.java index 63999fe891..8663a6e6d3 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/ActionRequestRestExecutorFactory.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/ActionRequestRestExecutorFactory.java @@ -13,13 +13,13 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor; -import com.amazon.opendistroforelasticsearch.sql.executor.csv.CSVResultRestExecutor; -import com.amazon.opendistroforelasticsearch.sql.executor.format.PrettyFormatRestExecutor; -import com.amazon.opendistroforelasticsearch.sql.query.QueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.join.ESJoinQueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.multi.MultiQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.csv.CSVResultRestExecutor; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.PrettyFormatRestExecutor; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.QueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.ESJoinQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.multi.MultiQueryAction; /** * Created by Eliran on 26/12/2015. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/AsyncRestExecutor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/AsyncRestExecutor.java similarity index 89% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/AsyncRestExecutor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/AsyncRestExecutor.java index 708942b137..76ec130a6c 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/AsyncRestExecutor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/AsyncRestExecutor.java @@ -13,15 +13,15 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor; - -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.metrics.MetricName; -import com.amazon.opendistroforelasticsearch.sql.metrics.Metrics; -import com.amazon.opendistroforelasticsearch.sql.query.QueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.join.BackOffRetryStrategy; -import com.amazon.opendistroforelasticsearch.sql.utils.LogUtils; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor; + +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.MetricName; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.Metrics; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.QueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.BackOffRetryStrategy; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.LogUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; @@ -38,7 +38,7 @@ import java.util.Map; import java.util.function.Predicate; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.QUERY_SLOWLOG; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.QUERY_SLOWLOG; /** * A RestExecutor wrapper to execute request asynchronously to avoid blocking transport thread. @@ -143,7 +143,7 @@ private void async(Client client, Map params, QueryAction queryA // Preserve context of calling thread to ensure headers of requests are forwarded when running blocking actions threadPool.schedule( - threadPool.preserveContext(LogUtils.withCurrentContext(runnable)), + LogUtils.withCurrentContext(runnable), new TimeValue(0L), SQL_WORKER_THREAD_POOL_NAME ); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/ElasticDefaultRestExecutor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/ElasticDefaultRestExecutor.java similarity index 87% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/ElasticDefaultRestExecutor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/ElasticDefaultRestExecutor.java index 8a11dbffc4..3ce18ea2f0 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/ElasticDefaultRestExecutor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/ElasticDefaultRestExecutor.java @@ -13,17 +13,17 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.executor.join.ElasticJoinExecutor; -import com.amazon.opendistroforelasticsearch.sql.executor.join.ElasticUtils; -import com.amazon.opendistroforelasticsearch.sql.executor.join.MetaSearchResult; -import com.amazon.opendistroforelasticsearch.sql.executor.multi.MultiRequestExecutorFactory; -import com.amazon.opendistroforelasticsearch.sql.query.QueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.SqlElasticRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.join.JoinRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.multi.MultiQueryRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.join.ElasticJoinExecutor; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.join.ElasticUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.join.MetaSearchResult; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.multi.MultiRequestExecutorFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.QueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.SqlElasticRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.JoinRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.multi.MultiQueryRequestBuilder; import com.google.common.collect.Maps; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/ElasticHitsExecutor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/ElasticHitsExecutor.java similarity index 84% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/ElasticHitsExecutor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/ElasticHitsExecutor.java index 5ea9af8d11..40a94ede3a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/ElasticHitsExecutor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/ElasticHitsExecutor.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; import org.elasticsearch.search.SearchHits; import java.io.IOException; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/ElasticResultHandler.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/ElasticResultHandler.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/ElasticResultHandler.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/ElasticResultHandler.java index 4ee8c5db43..dfc934db0c 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/ElasticResultHandler.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/ElasticResultHandler.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor; import org.elasticsearch.search.SearchHit; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/Format.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/Format.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/Format.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/Format.java index ab4ac5a56f..124d29cd95 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/Format.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/Format.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor; import com.google.common.collect.ImmutableMap; import lombok.Getter; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/GetIndexRequestRestListener.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/GetIndexRequestRestListener.java similarity index 88% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/GetIndexRequestRestListener.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/GetIndexRequestRestListener.java index a81bcf9429..09dd517278 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/GetIndexRequestRestListener.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/GetIndexRequestRestListener.java @@ -13,14 +13,14 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.SemanticAnalysisException; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.SemanticAnalysisException; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; -import org.elasticsearch.cluster.metadata.AliasMetaData; -import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; @@ -77,22 +77,22 @@ public RestResponse buildResponse(GetIndexResponse getIndexResponse, XContentBui return new BytesRestResponse(RestStatus.OK, builder); } - private void writeAliases(List aliases, XContentBuilder builder, ToXContent.Params params) + private void writeAliases(List aliases, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(Fields.ALIASES); if (aliases != null) { - for (AliasMetaData alias : aliases) { - AliasMetaData.Builder.toXContent(alias, builder, params); + for (AliasMetadata alias : aliases) { + AliasMetadata.Builder.toXContent(alias, builder, params); } } builder.endObject(); } - private void writeMappings(ImmutableOpenMap mappings, + private void writeMappings(ImmutableOpenMap mappings, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(Fields.MAPPINGS); if (mappings != null) { - for (ObjectObjectCursor typeEntry : mappings) { + for (ObjectObjectCursor typeEntry : mappings) { builder.field(typeEntry.key); builder.map(typeEntry.value.sourceAsMap()); } @@ -114,4 +114,4 @@ static class Fields { static final String SETTINGS = "settings"; static final String WARMERS = "warmers"; } -} \ No newline at end of file +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/QueryActionElasticExecutor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/QueryActionElasticExecutor.java similarity index 74% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/QueryActionElasticExecutor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/QueryActionElasticExecutor.java index cf98346643..fa42b1b056 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/QueryActionElasticExecutor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/QueryActionElasticExecutor.java @@ -13,25 +13,25 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.executor.join.ElasticJoinExecutor; -import com.amazon.opendistroforelasticsearch.sql.executor.multi.MultiRequestExecutorFactory; -import com.amazon.opendistroforelasticsearch.sql.executor.adapter.QueryPlanQueryAction; -import com.amazon.opendistroforelasticsearch.sql.executor.adapter.QueryPlanRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.expression.domain.BindingTuple; -import com.amazon.opendistroforelasticsearch.sql.query.AggregationQueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.DefaultQueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.DeleteQueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.DescribeQueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.QueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.ShowQueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.SqlElasticRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.SqlElasticSearchRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.join.ESJoinQueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.multi.MultiQueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.multi.MultiQueryRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.join.ElasticJoinExecutor; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.multi.MultiRequestExecutorFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.adapter.QueryPlanQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.adapter.QueryPlanRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.domain.BindingTuple; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.AggregationQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.DefaultQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.DeleteQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.DescribeQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.QueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.ShowQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.SqlElasticRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.SqlElasticSearchRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.ESJoinQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.multi.MultiQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.multi.MultiQueryRequestBuilder; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/RestExecutor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/RestExecutor.java similarity index 87% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/RestExecutor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/RestExecutor.java index 78d980ac47..c16b278025 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/RestExecutor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/RestExecutor.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor; -import com.amazon.opendistroforelasticsearch.sql.query.QueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.QueryAction; import org.elasticsearch.client.Client; import org.elasticsearch.rest.RestChannel; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/adapter/QueryPlanQueryAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/adapter/QueryPlanQueryAction.java similarity index 87% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/adapter/QueryPlanQueryAction.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/adapter/QueryPlanQueryAction.java index 6f3491186d..35c7e23672 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/adapter/QueryPlanQueryAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/adapter/QueryPlanQueryAction.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.adapter; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.adapter; -import com.amazon.opendistroforelasticsearch.sql.query.QueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.SqlElasticRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.QueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.SqlElasticRequestBuilder; import com.google.common.base.Strings; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/adapter/QueryPlanRequestBuilder.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/adapter/QueryPlanRequestBuilder.java similarity index 79% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/adapter/QueryPlanRequestBuilder.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/adapter/QueryPlanRequestBuilder.java index 1980876a4c..7a6080dd4e 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/adapter/QueryPlanRequestBuilder.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/adapter/QueryPlanRequestBuilder.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.adapter; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.adapter; -import com.amazon.opendistroforelasticsearch.sql.expression.domain.BindingTuple; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.ColumnNode; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.BindingTupleQueryPlanner; -import com.amazon.opendistroforelasticsearch.sql.query.SqlElasticRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.domain.BindingTuple; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.ColumnNode; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.BindingTupleQueryPlanner; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.SqlElasticRequestBuilder; import lombok.RequiredArgsConstructor; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestBuilder; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/csv/CSVResult.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/csv/CSVResult.java similarity index 97% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/csv/CSVResult.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/csv/CSVResult.java index dce1ec7229..805acbe7c4 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/csv/CSVResult.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/csv/CSVResult.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.csv; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.csv; import com.google.common.collect.ImmutableSet; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/csv/CSVResultRestExecutor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/csv/CSVResultRestExecutor.java similarity index 89% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/csv/CSVResultRestExecutor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/csv/CSVResultRestExecutor.java index 0b0520c6b0..cb7ef4259c 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/csv/CSVResultRestExecutor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/csv/CSVResultRestExecutor.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.csv; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.csv; -import com.amazon.opendistroforelasticsearch.sql.executor.QueryActionElasticExecutor; -import com.amazon.opendistroforelasticsearch.sql.executor.RestExecutor; -import com.amazon.opendistroforelasticsearch.sql.query.QueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.join.BackOffRetryStrategy; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.QueryActionElasticExecutor; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.RestExecutor; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.QueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.BackOffRetryStrategy; import com.google.common.base.Joiner; import org.elasticsearch.client.Client; import org.elasticsearch.rest.BytesRestResponse; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/csv/CSVResultsExtractor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/csv/CSVResultsExtractor.java similarity index 98% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/csv/CSVResultsExtractor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/csv/CSVResultsExtractor.java index 5fac2ad654..6574842a86 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/csv/CSVResultsExtractor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/csv/CSVResultsExtractor.java @@ -13,11 +13,11 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.csv; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.csv; -import com.amazon.opendistroforelasticsearch.sql.expression.domain.BindingTuple; -import com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValue; -import com.amazon.opendistroforelasticsearch.sql.utils.Util; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.domain.BindingTuple; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.Util; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/csv/CsvExtractorException.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/csv/CsvExtractorException.java similarity index 91% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/csv/CsvExtractorException.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/csv/CsvExtractorException.java index dbaf3e23ca..b397633acd 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/csv/CsvExtractorException.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/csv/CsvExtractorException.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.csv; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.csv; /** * Created by Eliran on 29/12/2015. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/cursor/CursorActionRequestRestExecutorFactory.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/cursor/CursorActionRequestRestExecutorFactory.java similarity index 89% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/cursor/CursorActionRequestRestExecutorFactory.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/cursor/CursorActionRequestRestExecutorFactory.java index 83ac6ee594..9901a878fc 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/cursor/CursorActionRequestRestExecutorFactory.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/cursor/CursorActionRequestRestExecutorFactory.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.cursor; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.cursor; -import com.amazon.opendistroforelasticsearch.sql.executor.Format; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.Format; import org.elasticsearch.rest.RestRequest; public class CursorActionRequestRestExecutorFactory { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/cursor/CursorAsyncRestExecutor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/cursor/CursorAsyncRestExecutor.java similarity index 87% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/cursor/CursorAsyncRestExecutor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/cursor/CursorAsyncRestExecutor.java index 84d19e945e..67aaa0f878 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/cursor/CursorAsyncRestExecutor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/cursor/CursorAsyncRestExecutor.java @@ -13,13 +13,13 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.cursor; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.cursor; -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; -import com.amazon.opendistroforelasticsearch.sql.metrics.MetricName; -import com.amazon.opendistroforelasticsearch.sql.metrics.Metrics; -import com.amazon.opendistroforelasticsearch.sql.query.join.BackOffRetryStrategy; -import com.amazon.opendistroforelasticsearch.sql.utils.LogUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.MetricName; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.Metrics; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.BackOffRetryStrategy; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.LogUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.client.Client; @@ -33,7 +33,7 @@ import java.time.Duration; import java.util.Map; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.QUERY_SLOWLOG; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.QUERY_SLOWLOG; public class CursorAsyncRestExecutor { /** @@ -93,7 +93,7 @@ private void async(Client client, Map params, RestChannel channe // Preserve context of calling thread to ensure headers of requests are forwarded when running blocking actions threadPool.schedule( - threadPool.preserveContext(LogUtils.withCurrentContext(runnable)), + LogUtils.withCurrentContext(runnable), new TimeValue(0L), SQL_WORKER_THREAD_POOL_NAME ); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/cursor/CursorCloseExecutor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/cursor/CursorCloseExecutor.java similarity index 89% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/cursor/CursorCloseExecutor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/cursor/CursorCloseExecutor.java index 43d2498d58..fc9d7c8681 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/cursor/CursorCloseExecutor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/cursor/CursorCloseExecutor.java @@ -13,13 +13,13 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.cursor; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.cursor; -import com.amazon.opendistroforelasticsearch.sql.cursor.CursorType; -import com.amazon.opendistroforelasticsearch.sql.cursor.DefaultCursor; -import com.amazon.opendistroforelasticsearch.sql.metrics.MetricName; -import com.amazon.opendistroforelasticsearch.sql.metrics.Metrics; -import com.amazon.opendistroforelasticsearch.sql.rewriter.matchtoterm.VerificationException; +import com.amazon.opendistroforelasticsearch.sql.legacy.cursor.CursorType; +import com.amazon.opendistroforelasticsearch.sql.legacy.cursor.DefaultCursor; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.MetricName; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.Metrics; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.matchtoterm.VerificationException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/cursor/CursorRestExecutor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/cursor/CursorRestExecutor.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/cursor/CursorRestExecutor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/cursor/CursorRestExecutor.java index 418841b967..3ff9463275 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/cursor/CursorRestExecutor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/cursor/CursorRestExecutor.java @@ -14,7 +14,7 @@ */ -package com.amazon.opendistroforelasticsearch.sql.executor.cursor; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.cursor; import org.elasticsearch.client.Client; import org.elasticsearch.rest.RestChannel; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/cursor/CursorResultExecutor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/cursor/CursorResultExecutor.java similarity index 87% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/cursor/CursorResultExecutor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/cursor/CursorResultExecutor.java index 131646ff1d..89b74d0283 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/cursor/CursorResultExecutor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/cursor/CursorResultExecutor.java @@ -13,16 +13,16 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.cursor; - -import com.amazon.opendistroforelasticsearch.sql.cursor.CursorType; -import com.amazon.opendistroforelasticsearch.sql.cursor.DefaultCursor; -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; -import com.amazon.opendistroforelasticsearch.sql.executor.Format; -import com.amazon.opendistroforelasticsearch.sql.executor.format.Protocol; -import com.amazon.opendistroforelasticsearch.sql.metrics.MetricName; -import com.amazon.opendistroforelasticsearch.sql.metrics.Metrics; -import com.amazon.opendistroforelasticsearch.sql.rewriter.matchtoterm.VerificationException; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.cursor; + +import com.amazon.opendistroforelasticsearch.sql.legacy.cursor.CursorType; +import com.amazon.opendistroforelasticsearch.sql.legacy.cursor.DefaultCursor; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.Format; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.Protocol; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.MetricName; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.Metrics; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.matchtoterm.VerificationException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; @@ -39,7 +39,7 @@ import java.util.Arrays; import java.util.Map; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.CURSOR_KEEPALIVE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.CURSOR_KEEPALIVE; import static org.elasticsearch.rest.RestStatus.OK; public class CursorResultExecutor implements CursorRestExecutor { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/BindingTupleResultSet.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/BindingTupleResultSet.java similarity index 66% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/BindingTupleResultSet.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/BindingTupleResultSet.java index a028db1ebc..4b164361fb 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/BindingTupleResultSet.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/BindingTupleResultSet.java @@ -13,13 +13,16 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.format; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.format; -import com.amazon.opendistroforelasticsearch.sql.expression.domain.BindingTuple; -import com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValue; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.ColumnNode; +import static com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.DateFieldFormatter.FORMAT_JDBC; + +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.domain.BindingTuple; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.ColumnNode; import com.google.common.annotations.VisibleForTesting; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -32,7 +35,7 @@ public class BindingTupleResultSet extends ResultSet { public BindingTupleResultSet(List columnNodes, List bindingTuples) { this.schema = buildSchema(columnNodes); - this.dataRows = buildDataRows(bindingTuples); + this.dataRows = buildDataRows(columnNodes, bindingTuples); } @VisibleForTesting @@ -47,12 +50,17 @@ public static Schema buildSchema(List columnNodes) { } @VisibleForTesting - public static DataRows buildDataRows(List bindingTuples) { + public static DataRows buildDataRows(List columnNodes, List bindingTuples) { List rowList = bindingTuples.stream().map(tuple -> { Map bindingMap = tuple.getBindingMap(); Map rowMap = new HashMap<>(); - for (String s : bindingMap.keySet()) { - rowMap.put(s, bindingMap.get(s).value()); + for (ColumnNode column : columnNodes) { + String columnName = column.columnName(); + Object value = bindingMap.get(columnName).value(); + if (column.getType() == Schema.Type.DATE) { + value = DateFormat.getFormattedDate(new Date((Long) value), FORMAT_JDBC); + } + rowMap.put(columnName, value); } return new DataRows.Row(rowMap); }).collect(Collectors.toList()); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DataRows.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/DataRows.java similarity index 97% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DataRows.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/DataRows.java index f7b766339d..c9783df8f5 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DataRows.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/DataRows.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.format; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.format; import java.util.Iterator; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/DateFieldFormatter.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/DateFieldFormatter.java index 30994b089d..01d5c26516 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatter.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/DateFieldFormatter.java @@ -13,11 +13,11 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.format; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.format; -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; -import com.amazon.opendistroforelasticsearch.sql.esdomain.mapping.FieldMappings; -import com.amazon.opendistroforelasticsearch.sql.esdomain.mapping.TypeMappings; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.mapping.FieldMappings; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.mapping.TypeMappings; import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang3.time.DateUtils; import org.apache.logging.log4j.LogManager; @@ -39,7 +39,7 @@ */ public class DateFieldFormatter { private static final Logger LOG = LogManager.getLogger(DateFieldFormatter.class); - private static final String FORMAT_JDBC = "yyyy-MM-dd HH:mm:ss.SSS"; + public static final String FORMAT_JDBC = "yyyy-MM-dd HH:mm:ss.SSS"; private static final String FORMAT_DELIMITER = "\\|\\|"; private static final String FORMAT_DOT_DATE_AND_TIME = "yyyy-MM-dd'T'HH:mm:ss.SSSZ"; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/DateFormat.java similarity index 98% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/DateFormat.java index 11606de281..4ab17e113c 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFormat.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/DateFormat.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.format; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.format; import java.time.Instant; import java.time.ZoneId; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DeleteResultSet.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/DeleteResultSet.java similarity index 92% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DeleteResultSet.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/DeleteResultSet.java index a40c57edfd..e7490897a8 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DeleteResultSet.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/DeleteResultSet.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.format; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.format; -import com.amazon.opendistroforelasticsearch.sql.domain.Delete; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Delete; import org.elasticsearch.client.Client; import org.elasticsearch.index.reindex.BulkByScrollResponse; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DescribeResultSet.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/DescribeResultSet.java similarity index 90% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DescribeResultSet.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/DescribeResultSet.java index 04044f8360..388b3259bd 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DescribeResultSet.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/DescribeResultSet.java @@ -13,16 +13,16 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.format; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.format; -import com.amazon.opendistroforelasticsearch.sql.domain.IndexStatement; -import com.amazon.opendistroforelasticsearch.sql.executor.format.DataRows.Row; -import com.amazon.opendistroforelasticsearch.sql.executor.format.Schema.Column; -import com.amazon.opendistroforelasticsearch.sql.executor.format.Schema.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.IndexStatement; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.DataRows.Row; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.Schema.Column; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.Schema.Type; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.collect.ImmutableOpenMap; import java.util.ArrayList; @@ -89,18 +89,18 @@ private List loadColumns() { private List loadRows() { List rows = new ArrayList<>(); GetIndexResponse indexResponse = (GetIndexResponse) queryResult; - ImmutableOpenMap> indexMappings = indexResponse.getMappings(); + ImmutableOpenMap> indexMappings = indexResponse.getMappings(); // Iterate through indices in indexMappings - for (ObjectObjectCursor> indexCursor : indexMappings) { + for (ObjectObjectCursor> indexCursor : indexMappings) { String index = indexCursor.key; // Check to see if index matches given pattern if (matchesPattern(index, statement.getIndexPattern())) { - ImmutableOpenMap typeMapping = indexCursor.value; + ImmutableOpenMap typeMapping = indexCursor.value; // Assuming ES 6.x, iterate through the only type of the index to get mapping data - for (ObjectObjectCursor typeCursor : typeMapping) { - MappingMetaData mappingMetaData = typeCursor.value; + for (ObjectObjectCursor typeCursor : typeMapping) { + MappingMetadata mappingMetaData = typeCursor.value; // Load rows for each field in the mapping rows.addAll(loadIndexData(index, mappingMetaData.getSourceAsMap())); } @@ -110,10 +110,10 @@ private List loadRows() { } @SuppressWarnings("unchecked") - private List loadIndexData(String index, Map mappingMetaData) { + private List loadIndexData(String index, Map mappingMetadata) { List rows = new ArrayList<>(); - Map flattenedMetaData = flattenMappingMetaData(mappingMetaData, "", new HashMap<>()); + Map flattenedMetaData = flattenMappingMetaData(mappingMetadata, "", new HashMap<>()); int position = 1; // Used as an arbitrary ORDINAL_POSITION value for the time being for (Entry entry : flattenedMetaData.entrySet()) { String columnPattern = statement.getColumnPattern(); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/ElasticsearchErrorMessage.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/ElasticsearchErrorMessage.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/ElasticsearchErrorMessage.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/ElasticsearchErrorMessage.java index e3dbfa46a2..a1f506eb37 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/ElasticsearchErrorMessage.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/ElasticsearchErrorMessage.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.format; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.format; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.ShardSearchFailure; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/ErrorMessage.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/ErrorMessage.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/ErrorMessage.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/ErrorMessage.java index bedbf949c1..4290dfa10e 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/ErrorMessage.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/ErrorMessage.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.format; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.format; import org.elasticsearch.rest.RestStatus; import org.json.JSONObject; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/ErrorMessageFactory.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/ErrorMessageFactory.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/ErrorMessageFactory.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/ErrorMessageFactory.java index a2116ae46c..d74dfb2a4a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/ErrorMessageFactory.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/ErrorMessageFactory.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.format; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.format; import org.elasticsearch.ElasticsearchException; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/PrettyFormatRestExecutor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java similarity index 86% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/PrettyFormatRestExecutor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java index 8b82ea4640..57a5f7c134 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/PrettyFormatRestExecutor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java @@ -13,16 +13,16 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.format; - -import com.amazon.opendistroforelasticsearch.sql.cursor.Cursor; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.executor.QueryActionElasticExecutor; -import com.amazon.opendistroforelasticsearch.sql.executor.RestExecutor; -import com.amazon.opendistroforelasticsearch.sql.cursor.DefaultCursor; -import com.amazon.opendistroforelasticsearch.sql.query.DefaultQueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.QueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.join.BackOffRetryStrategy; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.format; + +import com.amazon.opendistroforelasticsearch.sql.legacy.cursor.Cursor; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.QueryActionElasticExecutor; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.RestExecutor; +import com.amazon.opendistroforelasticsearch.sql.legacy.cursor.DefaultCursor; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.DefaultQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.QueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.BackOffRetryStrategy; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.search.SearchResponse; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/Protocol.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/Protocol.java similarity index 85% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/Protocol.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/Protocol.java index 9b03474964..bb6fdc48be 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/Protocol.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/Protocol.java @@ -13,26 +13,26 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.format; - -import com.amazon.opendistroforelasticsearch.sql.domain.ColumnTypeProvider; -import com.amazon.opendistroforelasticsearch.sql.domain.Delete; -import com.amazon.opendistroforelasticsearch.sql.domain.IndexStatement; -import com.amazon.opendistroforelasticsearch.sql.domain.Query; -import com.amazon.opendistroforelasticsearch.sql.domain.QueryStatement; -import com.amazon.opendistroforelasticsearch.sql.cursor.Cursor; -import com.amazon.opendistroforelasticsearch.sql.cursor.NullCursor; -import com.amazon.opendistroforelasticsearch.sql.executor.format.DataRows.Row; -import com.amazon.opendistroforelasticsearch.sql.executor.format.Schema.Column; -import com.amazon.opendistroforelasticsearch.sql.executor.adapter.QueryPlanQueryAction; -import com.amazon.opendistroforelasticsearch.sql.executor.adapter.QueryPlanRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.expression.domain.BindingTuple; -import com.amazon.opendistroforelasticsearch.sql.query.DefaultQueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.QueryAction; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.format; + +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.ColumnTypeProvider; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Delete; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.IndexStatement; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Query; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.QueryStatement; +import com.amazon.opendistroforelasticsearch.sql.legacy.cursor.Cursor; +import com.amazon.opendistroforelasticsearch.sql.legacy.cursor.NullCursor; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.DataRows.Row; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.Schema.Column; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.adapter.QueryPlanQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.adapter.QueryPlanRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.domain.BindingTuple; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.DefaultQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.QueryAction; import com.google.common.base.Strings; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.ColumnNode; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.ColumnNode; import org.elasticsearch.client.Client; import org.json.JSONArray; @@ -41,7 +41,7 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; -import static com.amazon.opendistroforelasticsearch.sql.domain.IndexStatement.StatementType; +import static com.amazon.opendistroforelasticsearch.sql.legacy.domain.IndexStatement.StatementType; public class Protocol { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/ResultSet.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/ResultSet.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/ResultSet.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/ResultSet.java index cbc5fb4b72..98450c8771 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/ResultSet.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/ResultSet.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.format; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.format; import org.elasticsearch.client.Client; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/Schema.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/Schema.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/Schema.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/Schema.java index 552f1e172b..b2b8994719 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/Schema.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/Schema.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.format; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.format; -import com.amazon.opendistroforelasticsearch.sql.domain.IndexStatement; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.IndexStatement; import java.util.HashSet; import java.util.Iterator; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/SelectResultSet.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/SelectResultSet.java index ece165a951..7a952061b5 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/SelectResultSet.java @@ -13,27 +13,27 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.format; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.format; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLCaseExpr; import com.alibaba.druid.sql.ast.expr.SQLCastExpr; import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; -import com.amazon.opendistroforelasticsearch.sql.domain.ColumnTypeProvider; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.domain.JoinSelect; -import com.amazon.opendistroforelasticsearch.sql.domain.MethodField; -import com.amazon.opendistroforelasticsearch.sql.domain.Query; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.domain.TableOnJoinSelect; -import com.amazon.opendistroforelasticsearch.sql.esdomain.mapping.FieldMapping; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlFeatureNotImplementedException; -import com.amazon.opendistroforelasticsearch.sql.executor.Format; -import com.amazon.opendistroforelasticsearch.sql.cursor.Cursor; -import com.amazon.opendistroforelasticsearch.sql.cursor.DefaultCursor; -import com.amazon.opendistroforelasticsearch.sql.metrics.MetricName; -import com.amazon.opendistroforelasticsearch.sql.metrics.Metrics; -import com.amazon.opendistroforelasticsearch.sql.utils.SQLFunctions; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.ColumnTypeProvider; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.JoinSelect; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.MethodField; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Query; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.TableOnJoinSelect; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.mapping.FieldMapping; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlFeatureNotImplementedException; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.Format; +import com.amazon.opendistroforelasticsearch.sql.legacy.cursor.Cursor; +import com.amazon.opendistroforelasticsearch.sql.legacy.cursor.DefaultCursor; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.MetricName; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.Metrics; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.SQLFunctions; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest; @@ -65,7 +65,7 @@ import static java.util.Collections.unmodifiableMap; import static java.util.stream.Collectors.toSet; -import static org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData; +import static org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetadata; public class SelectResultSet extends ResultSet { @@ -188,13 +188,13 @@ private void loadFromEsState(Query query) { .getFieldMappings(request) .actionGet(); - Map>> mappings = response.mappings(); + Map>> mappings = response.mappings(); if (mappings.isEmpty()) { throw new IllegalArgumentException(String.format("Index type %s does not exist", query.getFrom())); } // Assumption is all indices share the same mapping which is validated in TermFieldRewriter. - Map> indexMappings = mappings.values().iterator().next(); + Map> indexMappings = mappings.values().iterator().next(); // if index mappings size is 0 and the expression is a cast: that means that we are casting by alias // if so, add the original field that was being looked at to the mapping (how?) @@ -205,7 +205,7 @@ private void loadFromEsState(Query query) { * 2. If the incorrect type name was given then the response is null * 3. If no type name is given, the indexMapping is searched for a typeMapping */ - Map typeMappings = new HashMap<>(); + Map typeMappings = new HashMap<>(); if (indexMappings.containsKey(typeName)) { typeMappings = indexMappings.get(typeName); } else { @@ -416,7 +416,7 @@ private Schema.Type fetchMethodReturnType(int fieldIndex, MethodField field) { * will be used. */ private List populateColumns(Query query, String[] fieldNames, Map typeMappings) { + FieldMappingMetadata> typeMappings) { List fieldNameList; if (isSelectAll() || containsWildcard(query)) { @@ -911,4 +911,4 @@ private Map addMap(String field, Object term) { private boolean isJoinQuery() { return query instanceof JoinSelect; } -} \ No newline at end of file +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/ShowResultSet.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/ShowResultSet.java similarity index 88% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/ShowResultSet.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/ShowResultSet.java index 445ea76f7e..c6102b1da8 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/ShowResultSet.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/ShowResultSet.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.format; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.format; -import com.amazon.opendistroforelasticsearch.sql.domain.IndexStatement; -import com.amazon.opendistroforelasticsearch.sql.executor.format.DataRows.Row; -import com.amazon.opendistroforelasticsearch.sql.executor.format.Schema.Column; -import com.amazon.opendistroforelasticsearch.sql.executor.format.Schema.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.IndexStatement; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.DataRows.Row; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.Schema.Column; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.Schema.Type; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.client.Client; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/ElasticJoinExecutor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/join/ElasticJoinExecutor.java similarity index 90% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/ElasticJoinExecutor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/join/ElasticJoinExecutor.java index 7dcce0a1bb..c4826e0c14 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/ElasticJoinExecutor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/join/ElasticJoinExecutor.java @@ -13,17 +13,17 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.join; - -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.executor.ElasticHitsExecutor; -import com.amazon.opendistroforelasticsearch.sql.query.SqlElasticRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.join.HashJoinElasticRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.join.JoinRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.join.NestedLoopsElasticRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.join.TableInJoinRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.planner.HashJoinQueryPlanRequestBuilder; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.join; + +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.ElasticHitsExecutor; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.SqlElasticRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.HashJoinElasticRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.JoinRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.NestedLoopsElasticRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.TableInJoinRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.HashJoinQueryPlanRequestBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.search.TotalHits; @@ -31,6 +31,7 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -216,7 +217,10 @@ protected SearchHit createUnmachedResult(List secondTableReturnedFields, String unmatchedId = hit.getId() + "|0"; Text unamatchedType = new Text(hit.getType() + "|null"); - SearchHit searchHit = new SearchHit(docId, unmatchedId, unamatchedType, hit.getFields()); + Map documentFields = new HashMap<>(); + Map metaFields = new HashMap<>(); + SearchHit.splitFieldsByMetadata(hit.getFields(), documentFields, metaFields); + SearchHit searchHit = new SearchHit(docId, unmatchedId, unamatchedType, documentFields, metaFields); searchHit.sourceRef(hit.getSourceRef()); searchHit.getSourceAsMap().clear(); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/ElasticUtils.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/join/ElasticUtils.java similarity index 97% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/ElasticUtils.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/join/ElasticUtils.java index 6f2f07c184..42b0e31f1a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/ElasticUtils.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/join/ElasticUtils.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.join; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.join; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.query.join.BackOffRetryStrategy; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.BackOffRetryStrategy; import com.google.common.collect.ImmutableMap; import org.apache.lucene.search.TotalHits.Relation; import org.elasticsearch.action.search.SearchRequestBuilder; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/HashJoinComparisonStructure.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/join/HashJoinComparisonStructure.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/HashJoinComparisonStructure.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/join/HashJoinComparisonStructure.java index 7038da4ce9..cac75a60fe 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/HashJoinComparisonStructure.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/join/HashJoinComparisonStructure.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.join; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.join; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; import org.elasticsearch.search.SearchHit; import java.util.ArrayList; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/HashJoinElasticExecutor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/join/HashJoinElasticExecutor.java similarity index 92% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/HashJoinElasticExecutor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/join/HashJoinElasticExecutor.java index 1d796fa656..471ec86ef9 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/HashJoinElasticExecutor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/join/HashJoinElasticExecutor.java @@ -13,19 +13,20 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.join; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.join; import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.query.join.HashJoinElasticRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.join.TableInJoinRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.maker.QueryMaker; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.HashJoinElasticRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.TableInJoinRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.maker.QueryMaker; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.BoolQueryBuilder; @@ -182,9 +183,12 @@ private List createCombinedResults(TableInJoinRequestBuilder secondTa onlyReturnedFields(copiedSource, secondTableRequest.getReturnedFields(), secondTableRequest.getOriginalSelect().isSelectAll()); + Map documentFields = new HashMap<>(); + Map metaFields = new HashMap<>(); + SearchHit.splitFieldsByMetadata(matchingHit.getFields(), documentFields, metaFields); SearchHit searchHit = new SearchHit(matchingHit.docId(), combinedId, new Text(matchingHit.getType() + "|" + secondTableHit.getType()), - matchingHit.getFields()); + documentFields, metaFields); searchHit.sourceRef(matchingHit.getSourceRef()); searchHit.getSourceAsMap().clear(); searchHit.getSourceAsMap().putAll(matchingHit.getSourceAsMap()); @@ -239,7 +243,11 @@ private void createKeyToResultsAndFillOptimizationStructure( optimizationTermsFilterStructure.get(comparisonID)); //int docid , id - SearchHit searchHit = new SearchHit(resultIds, hit.getId(), new Text(hit.getType()), hit.getFields()); + Map documentFields = new HashMap<>(); + Map metaFields = new HashMap<>(); + SearchHit.splitFieldsByMetadata(hit.getFields(), documentFields, metaFields); + SearchHit searchHit = new SearchHit(resultIds, hit.getId(), new Text(hit.getType()), documentFields + , metaFields); searchHit.sourceRef(hit.getSourceRef()); onlyReturnedFields(searchHit.getSourceAsMap(), firstTableRequest.getReturnedFields(), diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/MetaSearchResult.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/join/MetaSearchResult.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/MetaSearchResult.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/join/MetaSearchResult.java index a037f48916..79bf7b9dcf 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/MetaSearchResult.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/join/MetaSearchResult.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.join; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.join; /** * Created by Eliran on 4/9/2015. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/NestedLoopsElasticExecutor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java similarity index 92% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/NestedLoopsElasticExecutor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java index badf5642fc..545d01db79 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/NestedLoopsElasticExecutor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java @@ -13,19 +13,19 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.join; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.join; import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource; -import com.amazon.opendistroforelasticsearch.sql.domain.Condition; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.esdomain.ESClient; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.query.DefaultQueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.join.BackOffRetryStrategy; -import com.amazon.opendistroforelasticsearch.sql.query.join.NestedLoopsElasticRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.join.TableInJoinRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.maker.Maker; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Condition; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.ESClient; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.DefaultQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.BackOffRetryStrategy; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.NestedLoopsElasticRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.TableInJoinRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.maker.Maker; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.search.MultiSearchRequest; @@ -33,12 +33,14 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -172,9 +174,12 @@ private SearchHit getMergedHit(int currentCombinedResults, String t1Alias, Strin SearchHit hitFromFirstTable, SearchHit matchedHit) { onlyReturnedFields(matchedHit.getSourceAsMap(), nestedLoopsRequest.getSecondTable().getReturnedFields(), nestedLoopsRequest.getSecondTable().getOriginalSelect().isSelectAll()); + Map documentFields = new HashMap<>(); + Map metaFields = new HashMap<>(); + SearchHit.splitFieldsByMetadata(hitFromFirstTable.getFields(), documentFields, metaFields); SearchHit searchHit = new SearchHit(currentCombinedResults, hitFromFirstTable.getId() + "|" + matchedHit.getId(), new Text(hitFromFirstTable.getType() + "|" + matchedHit.getType()), - hitFromFirstTable.getFields()); + documentFields, metaFields); searchHit.sourceRef(hitFromFirstTable.getSourceRef()); searchHit.getSourceAsMap().clear(); searchHit.getSourceAsMap().putAll(hitFromFirstTable.getSourceAsMap()); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/QueryPlanElasticExecutor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/join/QueryPlanElasticExecutor.java similarity index 87% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/QueryPlanElasticExecutor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/join/QueryPlanElasticExecutor.java index 4629884518..24d46c13fb 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/QueryPlanElasticExecutor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/join/QueryPlanElasticExecutor.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.join; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.join; -import com.amazon.opendistroforelasticsearch.sql.query.planner.HashJoinQueryPlanRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.QueryPlanner; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.HashJoinQueryPlanRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.QueryPlanner; import org.elasticsearch.search.SearchHit; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/SearchHitsResult.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/join/SearchHitsResult.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/SearchHitsResult.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/join/SearchHitsResult.java index 0fd5e66514..78c4e52564 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/SearchHitsResult.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/join/SearchHitsResult.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.join; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.join; import org.elasticsearch.search.SearchHit; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/ComperableHitResult.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/multi/ComperableHitResult.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/ComperableHitResult.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/multi/ComperableHitResult.java index 84e9a0c5bb..9c9755c13c 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/ComperableHitResult.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/multi/ComperableHitResult.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.multi; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.multi; -import com.amazon.opendistroforelasticsearch.sql.utils.Util; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.Util; import com.google.common.base.Joiner; import org.elasticsearch.search.SearchHit; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/MinusExecutor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/multi/MinusExecutor.java similarity index 92% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/MinusExecutor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/multi/MinusExecutor.java index 6441024f70..3063544050 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/MinusExecutor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/multi/MinusExecutor.java @@ -13,20 +13,20 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.multi; - -import com.amazon.opendistroforelasticsearch.sql.domain.Condition; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.domain.hints.Hint; -import com.amazon.opendistroforelasticsearch.sql.domain.hints.HintType; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.executor.ElasticHitsExecutor; -import com.amazon.opendistroforelasticsearch.sql.executor.join.ElasticUtils; -import com.amazon.opendistroforelasticsearch.sql.query.DefaultQueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.multi.MultiQueryRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.utils.Util; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.multi; + +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Condition; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints.Hint; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints.HintType; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.ElasticHitsExecutor; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.join.ElasticUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.DefaultQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.multi.MultiQueryRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.Util; import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.TotalHits.Relation; import org.elasticsearch.action.search.SearchResponse; @@ -134,7 +134,11 @@ private void fillMinusHitsFromOneField(String fieldName, Set fieldValues ArrayList values = new ArrayList<>(); values.add(result); fields.put(fieldName, new DocumentField(fieldName, values)); - SearchHit searchHit = new SearchHit(currentId, currentId + "", new Text(someHit.getType()), fields); + Map documentFields = new HashMap<>(); + Map metaFields = new HashMap<>(); + SearchHit.splitFieldsByMetadata(fields, documentFields, metaFields); + SearchHit searchHit = new SearchHit(currentId, currentId + "", new Text(someHit.getType()), + documentFields, metaFields); searchHit.sourceRef(someHit.getSourceRef()); searchHit.getSourceAsMap().clear(); Map sourceAsMap = new HashMap<>(); @@ -155,8 +159,11 @@ private void fillMinusHitsFromResults(Set comperableHitResu ArrayList values = new ArrayList<>(); values.add(result); SearchHit originalHit = result.getOriginalHit(); + Map documentFields = new HashMap<>(); + Map metaFields = new HashMap<>(); + SearchHit.splitFieldsByMetadata(originalHit.getFields(), documentFields, metaFields); SearchHit searchHit = new SearchHit(currentId, originalHit.getId(), new Text(originalHit.getType()), - originalHit.getFields()); + documentFields, metaFields); searchHit.sourceRef(originalHit.getSourceRef()); searchHit.getSourceAsMap().clear(); Map sourceAsMap = result.getFlattenMap(); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/MinusOneFieldAndOptimizationResult.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/multi/MinusOneFieldAndOptimizationResult.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/MinusOneFieldAndOptimizationResult.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/multi/MinusOneFieldAndOptimizationResult.java index 01bf32bea9..ce351ab1e0 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/MinusOneFieldAndOptimizationResult.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/multi/MinusOneFieldAndOptimizationResult.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.multi; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.multi; import org.elasticsearch.search.SearchHit; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/MultiRequestExecutorFactory.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/multi/MultiRequestExecutorFactory.java similarity index 78% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/MultiRequestExecutorFactory.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/multi/MultiRequestExecutorFactory.java index 3d52d9bf80..a23aaba3e6 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/MultiRequestExecutorFactory.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/multi/MultiRequestExecutorFactory.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.multi; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.multi; import com.alibaba.druid.sql.ast.statement.SQLUnionOperator; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.SemanticAnalysisException; -import com.amazon.opendistroforelasticsearch.sql.executor.ElasticHitsExecutor; -import com.amazon.opendistroforelasticsearch.sql.query.multi.MultiQueryRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.SemanticAnalysisException; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.ElasticHitsExecutor; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.multi.MultiQueryRequestBuilder; import org.elasticsearch.client.Client; /** diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/UnionExecutor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/multi/UnionExecutor.java similarity index 86% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/UnionExecutor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/multi/UnionExecutor.java index 0e1ed744cb..f133b71cb0 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/UnionExecutor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/multi/UnionExecutor.java @@ -13,21 +13,23 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.multi; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.multi; -import com.amazon.opendistroforelasticsearch.sql.executor.ElasticHitsExecutor; -import com.amazon.opendistroforelasticsearch.sql.query.multi.MultiQueryRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.utils.Util; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.ElasticHitsExecutor; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.multi.MultiQueryRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.Util; import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.TotalHits.Relation; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.text.Text; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -64,7 +66,10 @@ public void run() { private void fillInternalSearchHits(List unionHits, SearchHit[] hits, Map fieldNameToAlias) { for (SearchHit hit : hits) { - SearchHit searchHit = new SearchHit(currentId, hit.getId(), new Text(hit.getType()), hit.getFields()); + Map documentFields = new HashMap<>(); + Map metaFields = new HashMap<>(); + SearchHit.splitFieldsByMetadata(hit.getFields(), documentFields, metaFields); + SearchHit searchHit = new SearchHit(currentId, hit.getId(), new Text(hit.getType()), documentFields, metaFields); searchHit.sourceRef(hit.getSourceRef()); searchHit.getSourceAsMap().clear(); Map sourceAsMap = hit.getSourceAsMap(); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/Expression.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/Expression.java similarity index 77% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/Expression.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/Expression.java index 9b643ef325..e4d313786a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/Expression.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/Expression.java @@ -13,11 +13,11 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.core; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.core; -import com.amazon.opendistroforelasticsearch.sql.expression.domain.BindingTuple; -import com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.domain.BindingTuple; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValue; /** * The definition of the Expression. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/ExpressionFactory.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/ExpressionFactory.java similarity index 57% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/ExpressionFactory.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/ExpressionFactory.java index 1be172dfd4..4d03583ced 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/ExpressionFactory.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/ExpressionFactory.java @@ -13,40 +13,40 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.core; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.core; -import com.amazon.opendistroforelasticsearch.sql.expression.core.builder.ExpressionBuilder; -import com.amazon.opendistroforelasticsearch.sql.expression.core.builder.ArithmeticFunctionFactory; -import com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation; -import com.amazon.opendistroforelasticsearch.sql.expression.domain.BindingTuple; -import com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.builder.ExpressionBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.builder.ArithmeticFunctionFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.domain.BindingTuple; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValue; import com.google.common.collect.ImmutableMap; import java.util.List; import java.util.Map; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.ABS; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.ACOS; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.ADD; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.ASIN; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.ATAN; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.ATAN2; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.CBRT; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.CEIL; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.COS; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.COSH; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.DIVIDE; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.EXP; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.FLOOR; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.LN; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.LOG; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.LOG10; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.LOG2; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.MODULES; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.MULTIPLY; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.SUBTRACT; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.TAN; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.ABS; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.ACOS; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.ADD; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.ASIN; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.ATAN; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.ATAN2; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.CBRT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.CEIL; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.COS; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.COSH; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.DIVIDE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.EXP; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.FLOOR; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.LN; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.LOG; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.LOG10; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.LOG2; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.MODULES; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.MULTIPLY; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.SUBTRACT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.TAN; /** diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/builder/ArithmeticFunctionFactory.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/builder/ArithmeticFunctionFactory.java similarity index 91% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/builder/ArithmeticFunctionFactory.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/builder/ArithmeticFunctionFactory.java index d6faab5cbe..d85b340e8e 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/builder/ArithmeticFunctionFactory.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/builder/ArithmeticFunctionFactory.java @@ -13,13 +13,13 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.core.builder; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.builder; -import com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation; -import com.amazon.opendistroforelasticsearch.sql.expression.core.operator.BinaryScalarOperator; -import com.amazon.opendistroforelasticsearch.sql.expression.core.operator.DoubleBinaryScalarOperator; -import com.amazon.opendistroforelasticsearch.sql.expression.core.operator.DoubleUnaryScalarOperator; -import com.amazon.opendistroforelasticsearch.sql.expression.core.operator.UnaryScalarOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.BinaryScalarOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.DoubleBinaryScalarOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.DoubleUnaryScalarOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.UnaryScalarOperator; /** * The definition of arithmetic function builder factory. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/builder/BinaryExpressionBuilder.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/builder/BinaryExpressionBuilder.java similarity index 78% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/builder/BinaryExpressionBuilder.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/builder/BinaryExpressionBuilder.java index f6dff40ccd..f133e9985b 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/builder/BinaryExpressionBuilder.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/builder/BinaryExpressionBuilder.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.core.builder; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.builder; -import com.amazon.opendistroforelasticsearch.sql.expression.core.Expression; -import com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperator; -import com.amazon.opendistroforelasticsearch.sql.expression.domain.BindingTuple; -import com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.Expression; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.domain.BindingTuple; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValue; import lombok.RequiredArgsConstructor; import java.util.Arrays; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/builder/ExpressionBuilder.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/builder/ExpressionBuilder.java similarity index 82% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/builder/ExpressionBuilder.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/builder/ExpressionBuilder.java index 1f4af2fb3d..92b9f62fcb 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/builder/ExpressionBuilder.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/builder/ExpressionBuilder.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.core.builder; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.builder; -import com.amazon.opendistroforelasticsearch.sql.expression.core.Expression; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.Expression; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/builder/UnaryExpressionBuilder.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/builder/UnaryExpressionBuilder.java similarity index 78% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/builder/UnaryExpressionBuilder.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/builder/UnaryExpressionBuilder.java index ae89575411..695d59c61f 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/builder/UnaryExpressionBuilder.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/builder/UnaryExpressionBuilder.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.core.builder; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.builder; -import com.amazon.opendistroforelasticsearch.sql.expression.core.Expression; -import com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperator; -import com.amazon.opendistroforelasticsearch.sql.expression.domain.BindingTuple; -import com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.Expression; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.domain.BindingTuple; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValue; import lombok.RequiredArgsConstructor; import java.util.Arrays; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/operator/BinaryScalarOperator.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/operator/BinaryScalarOperator.java similarity index 72% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/operator/BinaryScalarOperator.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/operator/BinaryScalarOperator.java index bc0b3aca83..9f8c5da81c 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/operator/BinaryScalarOperator.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/operator/BinaryScalarOperator.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.core.operator; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator; -import com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValue; -import com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueFactory; import com.google.common.collect.ImmutableMap; import lombok.RequiredArgsConstructor; @@ -24,14 +24,14 @@ import java.util.Map; import java.util.function.BiFunction; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValue.ExprValueKind.DOUBLE_VALUE; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValue.ExprValueKind.FLOAT_VALUE; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValue.ExprValueKind.INTEGER_VALUE; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValue.ExprValueKind.LONG_VALUE; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueUtils.getDoubleValue; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueUtils.getFloatValue; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueUtils.getIntegerValue; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueUtils.getLongValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValue.ExprValueKind.DOUBLE_VALUE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValue.ExprValueKind.FLOAT_VALUE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValue.ExprValueKind.INTEGER_VALUE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValue.ExprValueKind.LONG_VALUE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueUtils.getDoubleValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueUtils.getFloatValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueUtils.getIntegerValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueUtils.getLongValue; /** * Binary Scalar Operator take two {@link ExprValue} as arguments ans return one {@link ExprValue} as result. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/operator/DoubleBinaryScalarOperator.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/operator/DoubleBinaryScalarOperator.java similarity index 82% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/operator/DoubleBinaryScalarOperator.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/operator/DoubleBinaryScalarOperator.java index d2f970cff3..3844941a69 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/operator/DoubleBinaryScalarOperator.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/operator/DoubleBinaryScalarOperator.java @@ -13,17 +13,17 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.core.operator; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator; -import com.amazon.opendistroforelasticsearch.sql.expression.model.ExprDoubleValue; -import com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValue; -import com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprDoubleValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueFactory; import lombok.RequiredArgsConstructor; import java.util.List; import java.util.function.BiFunction; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueUtils.getDoubleValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueUtils.getDoubleValue; /** * Double Binary Scalar Operator take two {@link ExprValue} which have double value as arguments ans return one diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/operator/DoubleUnaryScalarOperator.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/operator/DoubleUnaryScalarOperator.java similarity index 78% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/operator/DoubleUnaryScalarOperator.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/operator/DoubleUnaryScalarOperator.java index 00b3098b97..703fc448ca 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/operator/DoubleUnaryScalarOperator.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/operator/DoubleUnaryScalarOperator.java @@ -13,17 +13,17 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.core.operator; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator; -import com.amazon.opendistroforelasticsearch.sql.expression.model.ExprDoubleValue; -import com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValue; -import com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprDoubleValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueFactory; import lombok.RequiredArgsConstructor; import java.util.List; import java.util.function.Function; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueUtils.getDoubleValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueUtils.getDoubleValue; /** * Unary Binary Scalar Operator take one {@link ExprValue} which have double value as arguments ans return one diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/operator/ScalarOperation.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/operator/ScalarOperation.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/operator/ScalarOperation.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/operator/ScalarOperation.java index ea4d62fe2c..6df63f4959 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/operator/ScalarOperation.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/operator/ScalarOperation.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.core.operator; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/operator/ScalarOperator.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/operator/ScalarOperator.java similarity index 86% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/operator/ScalarOperator.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/operator/ScalarOperator.java index c8413e5508..1e92e8f06e 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/operator/ScalarOperator.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/operator/ScalarOperator.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.core.operator; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator; -import com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValue; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/operator/UnaryScalarOperator.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/operator/UnaryScalarOperator.java similarity index 75% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/operator/UnaryScalarOperator.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/operator/UnaryScalarOperator.java index 8c8b008894..b8da7832a4 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/core/operator/UnaryScalarOperator.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/core/operator/UnaryScalarOperator.java @@ -13,19 +13,19 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.core.operator; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator; -import com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValue; -import com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueFactory; import lombok.RequiredArgsConstructor; import java.util.List; import java.util.function.Function; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueUtils.getDoubleValue; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueUtils.getFloatValue; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueUtils.getIntegerValue; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueUtils.getLongValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueUtils.getDoubleValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueUtils.getFloatValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueUtils.getIntegerValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueUtils.getLongValue; /** * Unary Scalar Operator take one {@link ExprValue} as arguments ans return one {@link ExprValue} as result. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/domain/BindingTuple.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/domain/BindingTuple.java similarity index 86% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/domain/BindingTuple.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/domain/BindingTuple.java index 6e4435c3ee..857f6ddb02 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/domain/BindingTuple.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/domain/BindingTuple.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.domain; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.domain; -import com.amazon.opendistroforelasticsearch.sql.expression.model.ExprMissingValue; -import com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValue; -import com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprMissingValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueFactory; import lombok.Builder; import lombok.EqualsAndHashCode; import lombok.Getter; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprBooleanValue.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprBooleanValue.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprBooleanValue.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprBooleanValue.java index 212816231d..05802e3549 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprBooleanValue.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprBooleanValue.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.model; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.model; import lombok.EqualsAndHashCode; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprCollectionValue.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprCollectionValue.java similarity index 86% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprCollectionValue.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprCollectionValue.java index 5a2974008e..0eb91cca43 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprCollectionValue.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprCollectionValue.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.model; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.model; import lombok.EqualsAndHashCode; import lombok.RequiredArgsConstructor; @@ -21,7 +21,7 @@ import java.util.List; import java.util.stream.Collectors; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValue.ExprValueKind.COLLECTION_VALUE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValue.ExprValueKind.COLLECTION_VALUE; @EqualsAndHashCode @RequiredArgsConstructor diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprDoubleValue.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprDoubleValue.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprDoubleValue.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprDoubleValue.java index 827232cf68..6b5d78a557 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprDoubleValue.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprDoubleValue.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.model; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.model; import lombok.EqualsAndHashCode; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprFloatValue.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprFloatValue.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprFloatValue.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprFloatValue.java index 0a557a1ac3..762d3da371 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprFloatValue.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprFloatValue.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.model; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.model; import lombok.EqualsAndHashCode; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprIntegerValue.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprIntegerValue.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprIntegerValue.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprIntegerValue.java index 0736af7983..33be20ed0e 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprIntegerValue.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprIntegerValue.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.model; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.model; import lombok.EqualsAndHashCode; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprLongValue.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprLongValue.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprLongValue.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprLongValue.java index 542ea349be..65fedf57e9 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprLongValue.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprLongValue.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.model; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.model; import lombok.EqualsAndHashCode; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprMissingValue.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprMissingValue.java similarity index 91% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprMissingValue.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprMissingValue.java index 379a22168a..52c841a089 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprMissingValue.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprMissingValue.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.model; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.model; /** * The definition of the missing value. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprStringValue.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprStringValue.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprStringValue.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprStringValue.java index aaa4cafc65..efa75650dd 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprStringValue.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprStringValue.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.model; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.model; import lombok.EqualsAndHashCode; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprTupleValue.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprTupleValue.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprTupleValue.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprTupleValue.java index d96c835f32..6c7b4e1228 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprTupleValue.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprTupleValue.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.model; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.model; import lombok.EqualsAndHashCode; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprValue.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprValue.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprValue.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprValue.java index 78030376e5..0e16097ffb 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprValue.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprValue.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.model; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.model; /** * The definition of the Expression Value. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprValueFactory.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprValueFactory.java similarity index 91% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprValueFactory.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprValueFactory.java index 7254e4f3bf..e6e387bb0f 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprValueFactory.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprValueFactory.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.model; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.model; import java.util.ArrayList; import java.util.HashMap; @@ -41,6 +41,10 @@ public static ExprValue stringValue(String value) { return new ExprStringValue(value); } + public static ExprValue longValue(Long value) { + return new ExprLongValue(value); + } + public static ExprValue tupleValue(Map map) { Map valueMap = new HashMap<>(); map.forEach((k, v) -> valueMap.put(k, from(v))); @@ -61,7 +65,7 @@ public static ExprValue from(Object o) { } else if (o instanceof Integer) { return integerValue((Integer) o); } else if (o instanceof Long) { - return integerValue(((Long) o).intValue()); + return longValue(((Long) o)); } else if (o instanceof Boolean) { return booleanValue((Boolean) o); } else if (o instanceof Double) { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprValueUtils.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprValueUtils.java similarity index 82% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprValueUtils.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprValueUtils.java index 2007ca5d1d..b89032e372 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/model/ExprValueUtils.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/expression/model/ExprValueUtils.java @@ -13,17 +13,17 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.expression.model; +package com.amazon.opendistroforelasticsearch.sql.legacy.expression.model; import com.google.common.annotations.VisibleForTesting; import java.util.List; import java.util.Map; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValue.ExprValueKind.BOOLEAN_VALUE; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValue.ExprValueKind.COLLECTION_VALUE; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValue.ExprValueKind.STRING_VALUE; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValue.ExprValueKind.TUPLE_VALUE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValue.ExprValueKind.BOOLEAN_VALUE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValue.ExprValueKind.COLLECTION_VALUE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValue.ExprValueKind.STRING_VALUE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValue.ExprValueKind.TUPLE_VALUE; /** * The definition of ExprValue Utils. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/BasicCounter.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/BasicCounter.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/BasicCounter.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/BasicCounter.java index b9b4fc7e91..84bf624cfb 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/BasicCounter.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/BasicCounter.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.metrics; +package com.amazon.opendistroforelasticsearch.sql.legacy.metrics; import java.util.concurrent.atomic.LongAdder; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/Counter.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/Counter.java similarity index 91% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/Counter.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/Counter.java index 8ba911ee60..546876d228 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/Counter.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/Counter.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.metrics; +package com.amazon.opendistroforelasticsearch.sql.legacy.metrics; public interface Counter { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/GaugeMetric.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/GaugeMetric.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/GaugeMetric.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/GaugeMetric.java index a3b3a15310..93a9fb0f81 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/GaugeMetric.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/GaugeMetric.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.metrics; +package com.amazon.opendistroforelasticsearch.sql.legacy.metrics; import java.util.function.Supplier; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/Metric.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/Metric.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/Metric.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/Metric.java index 43b7e78239..adef60756f 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/Metric.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/Metric.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.metrics; +package com.amazon.opendistroforelasticsearch.sql.legacy.metrics; public abstract class Metric implements java.io.Serializable { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/MetricFactory.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/MetricFactory.java similarity index 90% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/MetricFactory.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/MetricFactory.java index 606838317f..7dfb55903f 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/MetricFactory.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/MetricFactory.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.metrics; +package com.amazon.opendistroforelasticsearch.sql.legacy.metrics; -import com.amazon.opendistroforelasticsearch.sql.query.join.BackOffRetryStrategy; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.BackOffRetryStrategy; public class MetricFactory { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/MetricName.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/MetricName.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/MetricName.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/MetricName.java index 843a082be6..9516f83e65 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/MetricName.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/MetricName.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.metrics; +package com.amazon.opendistroforelasticsearch.sql.legacy.metrics; import java.util.Arrays; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/Metrics.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/Metrics.java similarity index 97% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/Metrics.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/Metrics.java index 83eb5f3b38..e4e707966e 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/Metrics.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/Metrics.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.metrics; +package com.amazon.opendistroforelasticsearch.sql.legacy.metrics; import org.json.JSONObject; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/NumericMetric.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/NumericMetric.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/NumericMetric.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/NumericMetric.java index 9c902d71e3..2fafa64ac5 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/NumericMetric.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/NumericMetric.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.metrics; +package com.amazon.opendistroforelasticsearch.sql.legacy.metrics; public class NumericMetric extends Metric { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/RollingCounter.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/RollingCounter.java similarity index 92% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/RollingCounter.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/RollingCounter.java index 78df844a78..7475333756 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/metrics/RollingCounter.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/metrics/RollingCounter.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.metrics; +package com.amazon.opendistroforelasticsearch.sql.legacy.metrics; -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; -import com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings; import java.time.Clock; import java.util.concurrent.ConcurrentSkipListMap; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/CaseWhenParser.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/CaseWhenParser.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/CaseWhenParser.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/CaseWhenParser.java index 13d2ee2793..f429784962 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/CaseWhenParser.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/CaseWhenParser.java @@ -13,17 +13,17 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.parser; +package com.amazon.opendistroforelasticsearch.sql.legacy.parser; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOperator; import com.alibaba.druid.sql.ast.expr.SQLCaseExpr; import com.alibaba.druid.sql.ast.expr.SQLNullExpr; -import com.amazon.opendistroforelasticsearch.sql.domain.Condition; -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.utils.Util; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Condition; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.Util; import com.google.common.base.Joiner; import java.util.ArrayList; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/ChildrenType.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/ChildrenType.java similarity index 90% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/ChildrenType.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/ChildrenType.java index 289c242687..f1796305f4 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/ChildrenType.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/ChildrenType.java @@ -13,16 +13,16 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.parser; +package com.amazon.opendistroforelasticsearch.sql.legacy.parser; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr; import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; import com.alibaba.druid.sql.ast.expr.SQLTextLiteralExpr; -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.utils.Util; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.Util; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/ElasticLexer.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/ElasticLexer.java similarity index 97% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/ElasticLexer.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/ElasticLexer.java index 8e74cbfce3..771139d6b2 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/ElasticLexer.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/ElasticLexer.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.parser; +package com.amazon.opendistroforelasticsearch.sql.legacy.parser; import com.alibaba.druid.sql.dialect.mysql.parser.MySqlLexer; import com.alibaba.druid.sql.parser.ParserException; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/ElasticSqlExprParser.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/ElasticSqlExprParser.java similarity index 99% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/ElasticSqlExprParser.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/ElasticSqlExprParser.java index 709b73fdb8..b9d2985ac8 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/ElasticSqlExprParser.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/ElasticSqlExprParser.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.parser; +package com.amazon.opendistroforelasticsearch.sql.legacy.parser; import com.alibaba.druid.sql.ast.SQLCommentHint; import com.alibaba.druid.sql.ast.SQLDataType; @@ -61,7 +61,7 @@ import java.util.List; -import static com.amazon.opendistroforelasticsearch.sql.utils.StringUtils.isQuoted; +import static com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils.isQuoted; /** * Created by Eliran on 18/8/2015. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/ElasticSqlSelectParser.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/ElasticSqlSelectParser.java similarity index 98% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/ElasticSqlSelectParser.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/ElasticSqlSelectParser.java index e763e7facb..f62833babe 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/ElasticSqlSelectParser.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/ElasticSqlSelectParser.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.parser; +package com.amazon.opendistroforelasticsearch.sql.legacy.parser; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLSetQuantifier; @@ -36,7 +36,7 @@ import com.alibaba.druid.sql.parser.SQLExprParser; import com.alibaba.druid.sql.parser.SQLSelectParser; import com.alibaba.druid.sql.parser.Token; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlFeatureNotImplementedException; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlFeatureNotImplementedException; /** * Created by allwefantasy on 8/19/16. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/FieldMaker.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/FieldMaker.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/FieldMaker.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/FieldMaker.java index 2439b77976..5195d48ff6 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/FieldMaker.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/FieldMaker.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.parser; +package com.amazon.opendistroforelasticsearch.sql.legacy.parser; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLObject; @@ -34,15 +34,15 @@ import com.alibaba.druid.sql.ast.statement.SQLSelectGroupByClause; import com.alibaba.druid.sql.ast.statement.SQLSelectItem; import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.domain.KVValue; -import com.amazon.opendistroforelasticsearch.sql.domain.MethodField; -import com.amazon.opendistroforelasticsearch.sql.domain.ScriptMethodField; -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlFeatureNotImplementedException; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.utils.SQLFunctions; -import com.amazon.opendistroforelasticsearch.sql.utils.Util; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.KVValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.MethodField; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.ScriptMethodField; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlFeatureNotImplementedException; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.SQLFunctions; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.Util; import com.google.common.base.Strings; import org.elasticsearch.common.collect.Tuple; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/HavingParser.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/HavingParser.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/HavingParser.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/HavingParser.java index 4dd33fc635..d52375827a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/HavingParser.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/HavingParser.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.parser; +package com.amazon.opendistroforelasticsearch.sql.legacy.parser; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLObject; @@ -24,9 +24,9 @@ import com.alibaba.druid.sql.ast.expr.SQLInListExpr; import com.alibaba.druid.sql.ast.expr.SQLNotExpr; import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitorAdapter; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; import java.util.ArrayList; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/NestedType.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/NestedType.java similarity index 91% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/NestedType.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/NestedType.java index d09d55a1c0..b505b2dfab 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/NestedType.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/NestedType.java @@ -13,18 +13,18 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.parser; +package com.amazon.opendistroforelasticsearch.sql.legacy.parser; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr; import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; import com.alibaba.druid.sql.ast.expr.SQLTextLiteralExpr; -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.domain.bucketpath.BucketPath; -import com.amazon.opendistroforelasticsearch.sql.domain.bucketpath.Path; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.utils.Util; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.bucketpath.BucketPath; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.bucketpath.Path; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.Util; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/SQLOdbcExpr.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SQLOdbcExpr.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/SQLOdbcExpr.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SQLOdbcExpr.java index be980c771e..5e81dab116 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/SQLOdbcExpr.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SQLOdbcExpr.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.parser; +package com.amazon.opendistroforelasticsearch.sql.legacy.parser; import com.alibaba.druid.sql.ast.expr.SQLCharExpr; import com.alibaba.druid.sql.visitor.SQLASTVisitor; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/SQLParensIdentifierExpr.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SQLParensIdentifierExpr.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/SQLParensIdentifierExpr.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SQLParensIdentifierExpr.java index fb0ff04e2d..424e205e2c 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/SQLParensIdentifierExpr.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SQLParensIdentifierExpr.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.parser; +package com.amazon.opendistroforelasticsearch.sql.legacy.parser; import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/ScriptFilter.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/ScriptFilter.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/ScriptFilter.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/ScriptFilter.java index 5a2abf0950..ae79920f9b 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/ScriptFilter.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/ScriptFilter.java @@ -13,13 +13,13 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.parser; +package com.amazon.opendistroforelasticsearch.sql.legacy.parser; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.utils.Util; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.Util; import org.elasticsearch.script.ScriptType; import java.util.HashMap; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/SelectParser.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SelectParser.java similarity index 91% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/SelectParser.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SelectParser.java index 3f3a93bc4b..693f3a8a7d 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/SelectParser.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SelectParser.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.parser; +package com.amazon.opendistroforelasticsearch.sql.legacy.parser; /** * Created by allwefantasy on 9/2/16. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/SqlParser.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SqlParser.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/SqlParser.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SqlParser.java index 5cd8a087fb..d296f66b0c 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/SqlParser.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SqlParser.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.parser; +package com.amazon.opendistroforelasticsearch.sql.legacy.parser; import com.alibaba.druid.sql.ast.SQLCommentHint; import com.alibaba.druid.sql.ast.SQLExpr; @@ -37,20 +37,20 @@ import com.alibaba.druid.sql.ast.statement.SQLUnionQuery; import com.alibaba.druid.sql.dialect.mysql.ast.expr.MySqlSelectGroupByExpr; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock; -import com.amazon.opendistroforelasticsearch.sql.domain.Condition; -import com.amazon.opendistroforelasticsearch.sql.domain.Delete; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.domain.From; -import com.amazon.opendistroforelasticsearch.sql.domain.Having; -import com.amazon.opendistroforelasticsearch.sql.domain.JoinSelect; -import com.amazon.opendistroforelasticsearch.sql.domain.MethodField; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.domain.TableOnJoinSelect; -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.domain.hints.Hint; -import com.amazon.opendistroforelasticsearch.sql.domain.hints.HintFactory; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.query.multi.MultiQuerySelect; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Condition; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Delete; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.From; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Having; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.JoinSelect; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.MethodField; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.TableOnJoinSelect; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints.Hint; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints.HintFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.multi.MultiQuerySelect; import java.util.ArrayList; import java.util.HashMap; @@ -58,7 +58,7 @@ import java.util.Map; import java.util.stream.Collectors; -import static com.amazon.opendistroforelasticsearch.sql.utils.Util.NESTED_JOIN_TYPE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.utils.Util.NESTED_JOIN_TYPE; /** diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/SubQueryExpression.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SubQueryExpression.java similarity index 90% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/SubQueryExpression.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SubQueryExpression.java index fbeb50b95b..0830457148 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/SubQueryExpression.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SubQueryExpression.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.parser; +package com.amazon.opendistroforelasticsearch.sql.legacy.parser; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; /** * Created by Eliran on 3/10/2015. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/SubQueryParser.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SubQueryParser.java similarity index 90% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/SubQueryParser.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SubQueryParser.java index 7b0e19c947..f012a1572a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/SubQueryParser.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/SubQueryParser.java @@ -13,18 +13,18 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.parser; +package com.amazon.opendistroforelasticsearch.sql.legacy.parser; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.statement.SQLSelectItem; import com.alibaba.druid.sql.ast.statement.SQLSubqueryTableSource; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock; -import com.amazon.opendistroforelasticsearch.sql.domain.Condition; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.domain.Order; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Condition; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Order; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; import com.google.common.base.Strings; import java.util.HashMap; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/WhereParser.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/WhereParser.java similarity index 97% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/WhereParser.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/WhereParser.java index dccfdcaee5..73814fff0e 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/parser/WhereParser.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/parser/WhereParser.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.parser; +package com.amazon.opendistroforelasticsearch.sql.legacy.parser; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLBetweenExpr; @@ -35,16 +35,16 @@ import com.alibaba.druid.sql.ast.expr.SQLVariantRefExpr; import com.alibaba.druid.sql.ast.statement.SQLDeleteStatement; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock; -import com.amazon.opendistroforelasticsearch.sql.domain.Condition; -import com.amazon.opendistroforelasticsearch.sql.domain.KVValue; -import com.amazon.opendistroforelasticsearch.sql.domain.MethodField; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.query.maker.Maker; -import com.amazon.opendistroforelasticsearch.sql.spatial.SpatialParamsFactory; -import com.amazon.opendistroforelasticsearch.sql.utils.SQLFunctions; -import com.amazon.opendistroforelasticsearch.sql.utils.Util; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Condition; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.KVValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.MethodField; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.maker.Maker; +import com.amazon.opendistroforelasticsearch.sql.legacy.spatial.SpatialParamsFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.SQLFunctions; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.Util; import com.google.common.collect.Lists; import com.google.common.collect.Sets; diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/ElasticsearchSQLPluginConfig.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/ElasticsearchSQLPluginConfig.java new file mode 100644 index 0000000000..45be68dc0a --- /dev/null +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/ElasticsearchSQLPluginConfig.java @@ -0,0 +1,62 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy.plugin; + +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchNodeClient; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor.ElasticsearchExecutionEngine; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor.protector.ExecutionProtector; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor.protector.NoopExecutionProtector; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.ElasticsearchStorageEngine; +import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine; +import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.cluster.service.ClusterService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; + +/** + * Elasticsearch Plugin Config for SQL. + */ +public class ElasticsearchSQLPluginConfig { + @Autowired + private ClusterService clusterService; + + @Autowired + private NodeClient nodeClient; + + @Bean + public ElasticsearchClient client() { + return new ElasticsearchNodeClient(clusterService, nodeClient); + } + + @Bean + public StorageEngine storageEngine() { + return new ElasticsearchStorageEngine(client()); + } + + @Bean + public ExecutionEngine executionEngine() { + return new ElasticsearchExecutionEngine(client(), protector()); + } + + @Bean + public ExecutionProtector protector() { + return new NoopExecutionProtector(); + } +} diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java new file mode 100644 index 0000000000..6f243e684b --- /dev/null +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java @@ -0,0 +1,143 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy.plugin; + +import static com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.QueryResponse; +import static com.amazon.opendistroforelasticsearch.sql.protocol.response.format.JsonResponseFormatter.Style.PRETTY; +import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR; +import static org.elasticsearch.rest.RestStatus.OK; + +import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; +import com.amazon.opendistroforelasticsearch.sql.common.antlr.SyntaxCheckException; +import com.amazon.opendistroforelasticsearch.sql.common.response.ResponseListener; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.security.SecurityAccess; +import com.amazon.opendistroforelasticsearch.sql.protocol.response.QueryResult; +import com.amazon.opendistroforelasticsearch.sql.protocol.response.format.SimpleJsonResponseFormatter; +import com.amazon.opendistroforelasticsearch.sql.sql.SQLService; +import com.amazon.opendistroforelasticsearch.sql.sql.config.SQLServiceConfig; +import com.amazon.opendistroforelasticsearch.sql.sql.domain.SQLQueryRequest; +import java.io.IOException; +import java.security.PrivilegedExceptionAction; +import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.BytesRestResponse; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestStatus; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; + +/** + * New SQL REST action handler. This will not be registered to Elasticsearch unless: + * 1) we want to test new SQL engine; + * 2) all old functionalities migrated to new query engine and legacy REST handler removed. + */ +public class RestSQLQueryAction extends BaseRestHandler { + + private static final Logger LOG = LogManager.getLogger(); + + public static final RestChannelConsumer NOT_SUPPORTED_YET = null; + + private final ClusterService clusterService; + + public RestSQLQueryAction(ClusterService clusterService) { + super(); + this.clusterService = clusterService; + } + + @Override + public String getName() { + return "sql_query_action"; + } + + @Override + public List routes() { + throw new UnsupportedOperationException("New SQL handler is not ready yet"); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient nodeClient) { + throw new UnsupportedOperationException("New SQL handler is not ready yet"); + } + + /** + * Prepare REST channel consumer for a SQL query request. + * @param request SQL request + * @param nodeClient node client + * @return channel consumer + */ + public RestChannelConsumer prepareRequest(SQLQueryRequest request, NodeClient nodeClient) { + if (!request.isSupported()) { + return NOT_SUPPORTED_YET; + } + + SQLService sqlService = createSQLService(nodeClient); + UnresolvedPlan ast; + try { + ast = sqlService.parse(request.getQuery()); + } catch (SyntaxCheckException e) { + return NOT_SUPPORTED_YET; + } + return channel -> sqlService.execute(ast, createListener(channel)); + } + + private SQLService createSQLService(NodeClient client) { + return doPrivileged(() -> { + AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); + context.registerBean(ClusterService.class, () -> clusterService); + context.registerBean(NodeClient.class, () -> client); + context.register(ElasticsearchSQLPluginConfig.class); + context.register(SQLServiceConfig.class); + context.refresh(); + return context.getBean(SQLService.class); + }); + } + + // TODO: duplicate code here as in RestPPLQueryAction + private ResponseListener createListener(RestChannel channel) { + SimpleJsonResponseFormatter formatter = new SimpleJsonResponseFormatter(PRETTY); + return new ResponseListener() { + @Override + public void onResponse(QueryResponse response) { + sendResponse(OK, formatter.format(new QueryResult(response.getResults()))); + } + + @Override + public void onFailure(Exception e) { + LOG.error("Error happened during query handling", e); + sendResponse(INTERNAL_SERVER_ERROR, formatter.format(e)); + } + + private void sendResponse(RestStatus status, String content) { + channel.sendResponse(new BytesRestResponse( + status, "application/json; charset=UTF-8", content)); + } + }; + } + + private T doPrivileged(PrivilegedExceptionAction action) { + try { + return SecurityAccess.doPrivileged(action); + } catch (IOException e) { + throw new IllegalStateException("Failed to perform privileged action", e); + } + } + +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java similarity index 68% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlAction.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java index 51663fa3c8..b715dc8364 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java @@ -13,45 +13,46 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.plugin; +package com.amazon.opendistroforelasticsearch.sql.legacy.plugin; import com.alibaba.druid.sql.parser.ParserException; -import com.amazon.opendistroforelasticsearch.sql.antlr.OpenDistroSqlAnalyzer; -import com.amazon.opendistroforelasticsearch.sql.antlr.SqlAnalysisConfig; -import com.amazon.opendistroforelasticsearch.sql.antlr.SqlAnalysisException; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; -import com.amazon.opendistroforelasticsearch.sql.domain.ColumnTypeProvider; -import com.amazon.opendistroforelasticsearch.sql.domain.QueryActionRequest; -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; -import com.amazon.opendistroforelasticsearch.sql.exception.SQLFeatureDisabledException; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.executor.ActionRequestRestExecutorFactory; -import com.amazon.opendistroforelasticsearch.sql.executor.Format; -import com.amazon.opendistroforelasticsearch.sql.executor.RestExecutor; -import com.amazon.opendistroforelasticsearch.sql.executor.cursor.CursorActionRequestRestExecutorFactory; -import com.amazon.opendistroforelasticsearch.sql.executor.cursor.CursorAsyncRestExecutor; -import com.amazon.opendistroforelasticsearch.sql.executor.format.ErrorMessageFactory; -import com.amazon.opendistroforelasticsearch.sql.metrics.MetricName; -import com.amazon.opendistroforelasticsearch.sql.metrics.Metrics; -import com.amazon.opendistroforelasticsearch.sql.query.QueryAction; -import com.amazon.opendistroforelasticsearch.sql.request.SqlRequest; -import com.amazon.opendistroforelasticsearch.sql.request.SqlRequestFactory; -import com.amazon.opendistroforelasticsearch.sql.request.SqlRequestParam; -import com.amazon.opendistroforelasticsearch.sql.rewriter.matchtoterm.VerificationException; -import com.amazon.opendistroforelasticsearch.sql.utils.JsonPrettyFormatter; -import com.amazon.opendistroforelasticsearch.sql.utils.LogUtils; -import com.amazon.opendistroforelasticsearch.sql.utils.QueryDataAnonymizer; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.OpenDistroSqlAnalyzer; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.SqlAnalysisConfig; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.SqlAnalysisException; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.ColumnTypeProvider; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.QueryActionRequest; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SQLFeatureDisabledException; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.ActionRequestRestExecutorFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.Format; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.RestExecutor; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.cursor.CursorActionRequestRestExecutorFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.cursor.CursorAsyncRestExecutor; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.ErrorMessageFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.MetricName; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.Metrics; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.QueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.request.SqlRequest; +import com.amazon.opendistroforelasticsearch.sql.legacy.request.SqlRequestFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.request.SqlRequestParam; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.matchtoterm.VerificationException; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.JsonPrettyFormatter; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.LogUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.QueryDataAnonymizer; +import com.amazon.opendistroforelasticsearch.sql.sql.domain.SQLQueryRequest; import com.google.common.collect.ImmutableList; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.client.Client; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.BytesRestResponse; import org.elasticsearch.rest.RestChannel; -import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestStatus; @@ -66,10 +67,11 @@ import java.util.function.Predicate; import java.util.regex.Pattern; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.QUERY_ANALYSIS_ENABLED; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_SUGGESTION; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_THRESHOLD; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.SQL_ENABLED; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_ENABLED; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_SUGGESTION; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_THRESHOLD; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.SQL_ENABLED; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.SQL_NEW_ENGINE_ENABLED; import static org.elasticsearch.rest.RestStatus.BAD_REQUEST; import static org.elasticsearch.rest.RestStatus.OK; import static org.elasticsearch.rest.RestStatus.SERVICE_UNAVAILABLE; @@ -89,9 +91,15 @@ public class RestSqlAction extends BaseRestHandler { public static final String EXPLAIN_API_ENDPOINT = QUERY_API_ENDPOINT + "/_explain"; public static final String CURSOR_CLOSE_ENDPOINT = QUERY_API_ENDPOINT + "/close"; - RestSqlAction(Settings settings, RestController restController) { + /** + * New SQL query request handler. + */ + private final RestSQLQueryAction newSqlQueryHandler; + + public RestSqlAction(Settings settings, ClusterService clusterService) { super(); this.allowExplicitIndex = MULTI_ALLOW_EXPLICIT_INDEX.get(settings); + this.newSqlQueryHandler = new RestSQLQueryAction(clusterService); } @Override @@ -135,8 +143,25 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli LOG.info("[{}] Incoming request {}: {}", LogUtils.getRequestId(), request.uri(), QueryDataAnonymizer.anonymizeData(sqlRequest.getSql())); - final QueryAction queryAction = - explainRequest(client, sqlRequest, SqlRequestParam.getFormat(request.params())); + Format format = SqlRequestParam.getFormat(request.params()); + + if (isNewEngineEnabled()) { + // Route request to new query engine if it's supported already + SQLQueryRequest newSqlRequest = new SQLQueryRequest(sqlRequest.getJsonContent(), + sqlRequest.getSql(), + request.path(), + format.getFormatName()); + RestChannelConsumer result = newSqlQueryHandler.prepareRequest(newSqlRequest, client); + if (result != RestSQLQueryAction.NOT_SUPPORTED_YET) { + LOG.info("[{}] Request {} is handled by new SQL query engine", + LogUtils.getRequestId(), newSqlRequest); + return result; + } + LOG.debug("[{}] Request {} is not supported and falling back to old SQL engine", + LogUtils.getRequestId(), newSqlRequest); + } + + final QueryAction queryAction = explainRequest(client, sqlRequest, format); return channel -> executeSqlRequest(request, queryAction, client, channel); } catch (Exception e) { logAndPublishMetrics(e); @@ -237,6 +262,10 @@ private boolean isSQLFeatureEnabled() { return allowExplicitIndex && isSqlEnabled; } + private boolean isNewEngineEnabled() { + return LocalClusterState.state().getSettingValue(SQL_NEW_ENGINE_ENABLED); + } + private static ColumnTypeProvider performAnalysis(String sql) { LocalClusterState clusterState = LocalClusterState.state(); SqlAnalysisConfig config = new SqlAnalysisConfig( diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlSettingsAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlSettingsAction.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlSettingsAction.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlSettingsAction.java index d4174e06a1..8b935c6cf4 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlSettingsAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlSettingsAction.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.plugin; +package com.amazon.opendistroforelasticsearch.sql.legacy.plugin; -import com.amazon.opendistroforelasticsearch.sql.executor.format.ErrorMessageFactory; -import com.amazon.opendistroforelasticsearch.sql.utils.LogUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.ErrorMessageFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.LogUtils; import com.google.common.collect.ImmutableList; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlStatsAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlStatsAction.java similarity index 90% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlStatsAction.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlStatsAction.java index f4baab24b5..6e3c830d97 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlStatsAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlStatsAction.java @@ -13,11 +13,11 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.plugin; +package com.amazon.opendistroforelasticsearch.sql.legacy.plugin; -import com.amazon.opendistroforelasticsearch.sql.executor.format.ErrorMessageFactory; -import com.amazon.opendistroforelasticsearch.sql.metrics.Metrics; -import com.amazon.opendistroforelasticsearch.sql.utils.LogUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.ErrorMessageFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.Metrics; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.LogUtils; import com.google.common.collect.ImmutableList; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SearchDao.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SearchDao.java similarity index 80% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SearchDao.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SearchDao.java index 56f56903bc..a31e1fec40 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SearchDao.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SearchDao.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.plugin; +package com.amazon.opendistroforelasticsearch.sql.legacy.plugin; -import com.amazon.opendistroforelasticsearch.sql.domain.QueryActionRequest; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.query.ESActionFactory; -import com.amazon.opendistroforelasticsearch.sql.query.QueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.QueryActionRequest; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.ESActionFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.QueryAction; import org.elasticsearch.client.Client; import java.sql.SQLFeatureNotSupportedException; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SqlSettings.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SqlSettings.java similarity index 92% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SqlSettings.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SqlSettings.java index 795f4fe918..7bb9fe7261 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SqlSettings.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/SqlSettings.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.plugin; +package com.amazon.opendistroforelasticsearch.sql.legacy.plugin; -import com.amazon.opendistroforelasticsearch.sql.executor.Format; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.Format; import org.elasticsearch.common.settings.Setting; import java.util.ArrayList; @@ -39,6 +39,7 @@ public class SqlSettings { * 2) It has separate setting for Query and Fetch phase which are all ES internal concepts. */ public static final String SQL_ENABLED = "opendistro.sql.enabled"; + public static final String SQL_NEW_ENGINE_ENABLED = "opendistro.sql.engine.new.enabled"; public static final String QUERY_SLOWLOG = "opendistro.sql.query.slowlog"; public static final String QUERY_RESPONSE_FORMAT = "opendistro.sql.query.response.format"; public static final String QUERY_ANALYSIS_ENABLED = "opendistro.sql.query.analysis.enabled"; @@ -56,6 +57,7 @@ public class SqlSettings { public SqlSettings() { Map> settings = new HashMap<>(); settings.put(SQL_ENABLED, Setting.boolSetting(SQL_ENABLED, true, NodeScope, Dynamic)); + settings.put(SQL_NEW_ENGINE_ENABLED, Setting.boolSetting(SQL_NEW_ENGINE_ENABLED, false, NodeScope, Dynamic)); settings.put(QUERY_SLOWLOG, Setting.intSetting(QUERY_SLOWLOG, 2, NodeScope, Dynamic)); settings.put(QUERY_RESPONSE_FORMAT, Setting.simpleString(QUERY_RESPONSE_FORMAT, Format.JDBC.getFormatName(), NodeScope, Dynamic)); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/AggregationQueryAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/AggregationQueryAction.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/AggregationQueryAction.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/AggregationQueryAction.java index 6bc3d85b28..db75e7dd4f 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/AggregationQueryAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/AggregationQueryAction.java @@ -13,22 +13,22 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query; +package com.amazon.opendistroforelasticsearch.sql.legacy.query; import com.alibaba.druid.sql.ast.SQLExpr; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.domain.Having; -import com.amazon.opendistroforelasticsearch.sql.domain.KVValue; -import com.amazon.opendistroforelasticsearch.sql.domain.MethodField; -import com.amazon.opendistroforelasticsearch.sql.domain.Order; -import com.amazon.opendistroforelasticsearch.sql.domain.ScriptMethodField; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.domain.hints.Hint; -import com.amazon.opendistroforelasticsearch.sql.domain.hints.HintType; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.query.maker.AggMaker; -import com.amazon.opendistroforelasticsearch.sql.query.maker.QueryMaker; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Having; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.KVValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.MethodField; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Order; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.ScriptMethodField; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints.Hint; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints.HintType; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.maker.AggMaker; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.maker.QueryMaker; import com.google.common.collect.Lists; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequestBuilder; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/DefaultQueryAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/DefaultQueryAction.java similarity index 89% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/DefaultQueryAction.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/DefaultQueryAction.java index da92490ec0..0a4da06488 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/DefaultQueryAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/DefaultQueryAction.java @@ -13,27 +13,27 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query; +package com.amazon.opendistroforelasticsearch.sql.legacy.query; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOperator; import com.alibaba.druid.sql.ast.expr.SQLCastExpr; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.domain.KVValue; -import com.amazon.opendistroforelasticsearch.sql.domain.MethodField; -import com.amazon.opendistroforelasticsearch.sql.domain.Order; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.executor.Format; -import com.amazon.opendistroforelasticsearch.sql.executor.format.Schema; -import com.amazon.opendistroforelasticsearch.sql.metrics.MetricName; -import com.amazon.opendistroforelasticsearch.sql.metrics.Metrics; -import com.amazon.opendistroforelasticsearch.sql.query.maker.QueryMaker; -import com.amazon.opendistroforelasticsearch.sql.rewriter.nestedfield.NestedFieldProjection; -import com.amazon.opendistroforelasticsearch.sql.utils.SQLFunctions; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.KVValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.MethodField; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Order; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.Format; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.Schema; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.MetricName; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.Metrics; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.maker.QueryMaker; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.nestedfield.NestedFieldProjection; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.SQLFunctions; import com.google.common.annotations.VisibleForTesting; import org.elasticsearch.action.search.SearchAction; @@ -60,8 +60,8 @@ import java.util.Objects; import java.util.Optional; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.CURSOR_ENABLED; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.CURSOR_KEEPALIVE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.CURSOR_ENABLED; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.CURSOR_KEEPALIVE; /** * Transform SQL query to standard Elasticsearch search query diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/DeleteQueryAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/DeleteQueryAction.java similarity index 87% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/DeleteQueryAction.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/DeleteQueryAction.java index 6b0f480f50..ebf9e5f5df 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/DeleteQueryAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/DeleteQueryAction.java @@ -13,13 +13,13 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query; +package com.amazon.opendistroforelasticsearch.sql.legacy.query; -import com.amazon.opendistroforelasticsearch.sql.domain.Delete; -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.query.maker.QueryMaker; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Delete; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.maker.QueryMaker; import org.elasticsearch.client.Client; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/DescribeQueryAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/DescribeQueryAction.java similarity index 82% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/DescribeQueryAction.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/DescribeQueryAction.java index 72f201e690..12926a81d1 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/DescribeQueryAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/DescribeQueryAction.java @@ -13,11 +13,11 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query; +package com.amazon.opendistroforelasticsearch.sql.legacy.query; -import com.amazon.opendistroforelasticsearch.sql.domain.IndexStatement; -import com.amazon.opendistroforelasticsearch.sql.domain.QueryStatement; -import com.amazon.opendistroforelasticsearch.sql.utils.Util; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.IndexStatement; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.QueryStatement; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.Util; import org.elasticsearch.action.admin.indices.get.GetIndexRequestBuilder; import org.elasticsearch.client.Client; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/ESActionFactory.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/ESActionFactory.java similarity index 76% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/ESActionFactory.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/ESActionFactory.java index 4251362022..ac05da3236 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/ESActionFactory.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/ESActionFactory.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query; +package com.amazon.opendistroforelasticsearch.sql.legacy.query; import com.alibaba.druid.sql.ast.expr.SQLAggregateExpr; import com.alibaba.druid.sql.ast.expr.SQLAllColumnExpr; @@ -28,36 +28,36 @@ import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitorAdapter; import com.alibaba.druid.sql.parser.SQLStatementParser; -import com.amazon.opendistroforelasticsearch.sql.domain.ColumnTypeProvider; -import com.amazon.opendistroforelasticsearch.sql.domain.Delete; -import com.amazon.opendistroforelasticsearch.sql.domain.IndexStatement; -import com.amazon.opendistroforelasticsearch.sql.domain.JoinSelect; -import com.amazon.opendistroforelasticsearch.sql.domain.QueryActionRequest; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.executor.ElasticResultHandler; -import com.amazon.opendistroforelasticsearch.sql.executor.Format; -import com.amazon.opendistroforelasticsearch.sql.executor.QueryActionElasticExecutor; -import com.amazon.opendistroforelasticsearch.sql.executor.adapter.QueryPlanQueryAction; -import com.amazon.opendistroforelasticsearch.sql.executor.adapter.QueryPlanRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.parser.ElasticLexer; -import com.amazon.opendistroforelasticsearch.sql.parser.SqlParser; -import com.amazon.opendistroforelasticsearch.sql.parser.SubQueryExpression; -import com.amazon.opendistroforelasticsearch.sql.query.join.ESJoinQueryActionFactory; -import com.amazon.opendistroforelasticsearch.sql.query.multi.MultiQueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.multi.MultiQuerySelect; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.BindingTupleQueryPlanner; -import com.amazon.opendistroforelasticsearch.sql.rewriter.RewriteRuleExecutor; -import com.amazon.opendistroforelasticsearch.sql.rewriter.alias.TableAliasPrefixRemoveRule; -import com.amazon.opendistroforelasticsearch.sql.rewriter.identifier.UnquoteIdentifierRule; -import com.amazon.opendistroforelasticsearch.sql.rewriter.join.JoinRewriteRule; -import com.amazon.opendistroforelasticsearch.sql.rewriter.matchtoterm.TermFieldRewriter; -import com.amazon.opendistroforelasticsearch.sql.rewriter.matchtoterm.TermFieldRewriter.TermRewriterFilter; -import com.amazon.opendistroforelasticsearch.sql.rewriter.nestedfield.NestedFieldRewriter; -import com.amazon.opendistroforelasticsearch.sql.rewriter.ordinal.OrdinalRewriterRule; -import com.amazon.opendistroforelasticsearch.sql.rewriter.parent.SQLExprParentSetterRule; -import com.amazon.opendistroforelasticsearch.sql.rewriter.subquery.SubQueryRewriteRule; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.ColumnTypeProvider; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Delete; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.IndexStatement; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.JoinSelect; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.QueryActionRequest; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.ElasticResultHandler; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.Format; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.QueryActionElasticExecutor; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.adapter.QueryPlanQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.adapter.QueryPlanRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.ElasticLexer; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.SqlParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.SubQueryExpression; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.ESJoinQueryActionFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.multi.MultiQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.multi.MultiQuerySelect; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.BindingTupleQueryPlanner; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.RewriteRuleExecutor; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.alias.TableAliasPrefixRemoveRule; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.identifier.UnquoteIdentifierRule; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.join.JoinRewriteRule; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.matchtoterm.TermFieldRewriter; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.matchtoterm.TermFieldRewriter.TermRewriterFilter; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.nestedfield.NestedFieldRewriter; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.ordinal.OrdinalRewriterRule; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.parent.SQLExprParentSetterRule; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery.SubQueryRewriteRule; import com.google.common.annotations.VisibleForTesting; import org.elasticsearch.client.Client; import org.elasticsearch.search.SearchHit; @@ -67,8 +67,8 @@ import java.util.ArrayList; import java.util.List; -import static com.amazon.opendistroforelasticsearch.sql.domain.IndexStatement.StatementType; -import static com.amazon.opendistroforelasticsearch.sql.utils.Util.toSqlExpr; +import static com.amazon.opendistroforelasticsearch.sql.legacy.domain.IndexStatement.StatementType; +import static com.amazon.opendistroforelasticsearch.sql.legacy.utils.Util.toSqlExpr; public class ESActionFactory { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/QueryAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/QueryAction.java similarity index 92% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/QueryAction.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/QueryAction.java index ee7e9ceaf7..04bd92c84a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/QueryAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/QueryAction.java @@ -13,17 +13,17 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query; - -import com.amazon.opendistroforelasticsearch.sql.domain.ColumnTypeProvider; -import com.amazon.opendistroforelasticsearch.sql.domain.Query; -import com.amazon.opendistroforelasticsearch.sql.domain.QueryStatement; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.domain.hints.Hint; -import com.amazon.opendistroforelasticsearch.sql.domain.hints.HintType; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.executor.Format; -import com.amazon.opendistroforelasticsearch.sql.request.SqlRequest; +package com.amazon.opendistroforelasticsearch.sql.legacy.query; + +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.ColumnTypeProvider; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Query; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.QueryStatement; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints.Hint; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints.HintType; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.Format; +import com.amazon.opendistroforelasticsearch.sql.legacy.request.SqlRequest; import com.fasterxml.jackson.core.JsonFactory; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.support.IndicesOptions; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/ShowQueryAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/ShowQueryAction.java similarity index 80% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/ShowQueryAction.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/ShowQueryAction.java index 94052a83dd..986c65b954 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/ShowQueryAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/ShowQueryAction.java @@ -13,14 +13,14 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query; +package com.amazon.opendistroforelasticsearch.sql.legacy.query; -import com.amazon.opendistroforelasticsearch.sql.domain.IndexStatement; -import com.amazon.opendistroforelasticsearch.sql.domain.QueryStatement; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.IndexStatement; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.QueryStatement; import org.elasticsearch.action.admin.indices.get.GetIndexRequestBuilder; import org.elasticsearch.client.Client; -import static com.amazon.opendistroforelasticsearch.sql.utils.Util.prepareIndexRequestBuilder; +import static com.amazon.opendistroforelasticsearch.sql.legacy.utils.Util.prepareIndexRequestBuilder; public class ShowQueryAction extends QueryAction { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/SqlElasticDeleteByQueryRequestBuilder.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/SqlElasticDeleteByQueryRequestBuilder.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/SqlElasticDeleteByQueryRequestBuilder.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/SqlElasticDeleteByQueryRequestBuilder.java index 2c962217d1..4e3a1e92b1 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/SqlElasticDeleteByQueryRequestBuilder.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/SqlElasticDeleteByQueryRequestBuilder.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query; +package com.amazon.opendistroforelasticsearch.sql.legacy.query; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestBuilder; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/SqlElasticRequestBuilder.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/SqlElasticRequestBuilder.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/SqlElasticRequestBuilder.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/SqlElasticRequestBuilder.java index 09721a7a61..6b1391e528 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/SqlElasticRequestBuilder.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/SqlElasticRequestBuilder.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query; +package com.amazon.opendistroforelasticsearch.sql.legacy.query; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestBuilder; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/SqlElasticSearchRequestBuilder.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/SqlElasticSearchRequestBuilder.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/SqlElasticSearchRequestBuilder.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/SqlElasticSearchRequestBuilder.java index 1590f47b66..92527ab452 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/SqlElasticSearchRequestBuilder.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/SqlElasticSearchRequestBuilder.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query; +package com.amazon.opendistroforelasticsearch.sql.legacy.query; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestBuilder; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/BackOffRetryStrategy.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/BackOffRetryStrategy.java similarity index 97% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/BackOffRetryStrategy.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/BackOffRetryStrategy.java index c7054081f8..41a3852619 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/BackOffRetryStrategy.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/BackOffRetryStrategy.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.join; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.join; -import com.amazon.opendistroforelasticsearch.sql.metrics.MetricName; -import com.amazon.opendistroforelasticsearch.sql.metrics.Metrics; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.MetricName; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.Metrics; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.common.collect.Tuple; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/ESHashJoinQueryAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/ESHashJoinQueryAction.java similarity index 89% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/ESHashJoinQueryAction.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/ESHashJoinQueryAction.java index 12a8f3969d..c1180d5c57 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/ESHashJoinQueryAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/ESHashJoinQueryAction.java @@ -13,16 +13,16 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.join; - -import com.amazon.opendistroforelasticsearch.sql.domain.Condition; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.domain.JoinSelect; -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.domain.hints.Hint; -import com.amazon.opendistroforelasticsearch.sql.domain.hints.HintType; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.query.planner.HashJoinQueryPlanRequestBuilder; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.join; + +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Condition; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.JoinSelect; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints.Hint; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints.HintType; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.HashJoinQueryPlanRequestBuilder; import org.elasticsearch.client.Client; import java.util.AbstractMap; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/ESJoinQueryAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/ESJoinQueryAction.java similarity index 84% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/ESJoinQueryAction.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/ESJoinQueryAction.java index 331ef15548..4f83e49ac2 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/ESJoinQueryAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/ESJoinQueryAction.java @@ -13,19 +13,19 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.join; - -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.domain.JoinSelect; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.domain.TableOnJoinSelect; -import com.amazon.opendistroforelasticsearch.sql.domain.hints.Hint; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.query.DefaultQueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.QueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.SqlElasticRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.planner.HashJoinQueryPlanRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.Config; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.join; + +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.JoinSelect; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.TableOnJoinSelect; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints.Hint; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.DefaultQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.QueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.SqlElasticRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.HashJoinQueryPlanRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.Config; import org.elasticsearch.client.Client; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/ESJoinQueryActionFactory.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/ESJoinQueryActionFactory.java similarity index 79% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/ESJoinQueryActionFactory.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/ESJoinQueryActionFactory.java index 67ac835cc1..3774dcbcca 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/ESJoinQueryActionFactory.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/ESJoinQueryActionFactory.java @@ -13,13 +13,13 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.join; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.join; -import com.amazon.opendistroforelasticsearch.sql.domain.Condition; -import com.amazon.opendistroforelasticsearch.sql.domain.JoinSelect; -import com.amazon.opendistroforelasticsearch.sql.domain.hints.Hint; -import com.amazon.opendistroforelasticsearch.sql.domain.hints.HintType; -import com.amazon.opendistroforelasticsearch.sql.query.QueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Condition; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.JoinSelect; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints.Hint; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints.HintType; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.QueryAction; import org.elasticsearch.client.Client; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/ESNestedLoopsQueryAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/ESNestedLoopsQueryAction.java similarity index 82% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/ESNestedLoopsQueryAction.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/ESNestedLoopsQueryAction.java index 204ff51280..640c4a671d 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/ESNestedLoopsQueryAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/ESNestedLoopsQueryAction.java @@ -13,13 +13,13 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.join; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.join; -import com.amazon.opendistroforelasticsearch.sql.domain.JoinSelect; -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.domain.hints.Hint; -import com.amazon.opendistroforelasticsearch.sql.domain.hints.HintType; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.JoinSelect; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints.Hint; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints.HintType; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; import org.elasticsearch.client.Client; /** diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/HashJoinElasticRequestBuilder.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/HashJoinElasticRequestBuilder.java similarity index 92% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/HashJoinElasticRequestBuilder.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/HashJoinElasticRequestBuilder.java index 61fbb45de3..fcf60e200d 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/HashJoinElasticRequestBuilder.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/HashJoinElasticRequestBuilder.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.join; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.join; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; import java.util.List; import java.util.Map; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/JoinRequestBuilder.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/JoinRequestBuilder.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/JoinRequestBuilder.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/JoinRequestBuilder.java index 12d7dac1b2..6d533ce2a6 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/JoinRequestBuilder.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/JoinRequestBuilder.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.join; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.join; import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource; -import com.amazon.opendistroforelasticsearch.sql.query.SqlElasticRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.SqlElasticRequestBuilder; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.ActionResponse; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/NestedLoopsElasticRequestBuilder.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/NestedLoopsElasticRequestBuilder.java similarity index 90% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/NestedLoopsElasticRequestBuilder.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/NestedLoopsElasticRequestBuilder.java index af5dc92aa0..14c806f327 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/NestedLoopsElasticRequestBuilder.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/NestedLoopsElasticRequestBuilder.java @@ -13,13 +13,13 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.join; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.join; -import com.amazon.opendistroforelasticsearch.sql.domain.Condition; -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.query.maker.QueryMaker; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Condition; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.maker.QueryMaker; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/TableInJoinRequestBuilder.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/TableInJoinRequestBuilder.java similarity index 90% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/TableInJoinRequestBuilder.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/TableInJoinRequestBuilder.java index 37870319c3..b269963b57 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/join/TableInJoinRequestBuilder.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/join/TableInJoinRequestBuilder.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.join; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.join; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; import org.elasticsearch.action.search.SearchRequestBuilder; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/maker/AggMaker.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/maker/AggMaker.java similarity index 97% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/maker/AggMaker.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/maker/AggMaker.java index a7afbaefec..5d4549b9d6 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/maker/AggMaker.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/maker/AggMaker.java @@ -13,20 +13,20 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.maker; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.maker; import com.alibaba.druid.sql.ast.expr.SQLAggregateOption; -import com.amazon.opendistroforelasticsearch.sql.domain.Condition; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.domain.KVValue; -import com.amazon.opendistroforelasticsearch.sql.domain.MethodField; -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.domain.Where.CONN; -import com.amazon.opendistroforelasticsearch.sql.domain.bucketpath.Path; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.parser.ChildrenType; -import com.amazon.opendistroforelasticsearch.sql.parser.NestedType; -import com.amazon.opendistroforelasticsearch.sql.utils.Util; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Condition; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.KVValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.MethodField; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where.CONN; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.bucketpath.Path; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.ChildrenType; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.NestedType; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.Util; import com.fasterxml.jackson.core.JsonFactory; import org.apache.commons.lang3.StringUtils; import org.elasticsearch.common.ParsingException; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/maker/Maker.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/maker/Maker.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/maker/Maker.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/maker/Maker.java index 7dd095a8be..1a711e533c 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/maker/Maker.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/maker/Maker.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.maker; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.maker; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; @@ -22,18 +22,18 @@ import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr; import com.alibaba.druid.sql.ast.expr.SQLNumericLiteralExpr; -import com.amazon.opendistroforelasticsearch.sql.domain.Condition; -import com.amazon.opendistroforelasticsearch.sql.domain.Paramer; -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.parser.ScriptFilter; -import com.amazon.opendistroforelasticsearch.sql.parser.SubQueryExpression; -import com.amazon.opendistroforelasticsearch.sql.spatial.BoundingBoxFilterParams; -import com.amazon.opendistroforelasticsearch.sql.spatial.DistanceFilterParams; -import com.amazon.opendistroforelasticsearch.sql.spatial.Point; -import com.amazon.opendistroforelasticsearch.sql.spatial.PolygonFilterParams; -import com.amazon.opendistroforelasticsearch.sql.spatial.WktToGeoJsonConverter; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Condition; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Paramer; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.ScriptFilter; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.SubQueryExpression; +import com.amazon.opendistroforelasticsearch.sql.legacy.spatial.BoundingBoxFilterParams; +import com.amazon.opendistroforelasticsearch.sql.legacy.spatial.DistanceFilterParams; +import com.amazon.opendistroforelasticsearch.sql.legacy.spatial.Point; +import com.amazon.opendistroforelasticsearch.sql.legacy.spatial.PolygonFilterParams; +import com.amazon.opendistroforelasticsearch.sql.legacy.spatial.WktToGeoJsonConverter; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import org.apache.lucene.search.join.ScoreMode; @@ -72,7 +72,7 @@ import java.util.Map; import java.util.Set; -import static com.amazon.opendistroforelasticsearch.sql.parser.WhereParser.getConditionForMethod; +import static com.amazon.opendistroforelasticsearch.sql.legacy.parser.WhereParser.getConditionForMethod; public abstract class Maker { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/maker/QueryMaker.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/maker/QueryMaker.java similarity index 92% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/maker/QueryMaker.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/maker/QueryMaker.java index 78aa90fe1e..6df498180a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/maker/QueryMaker.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/maker/QueryMaker.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.maker; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.maker; -import com.amazon.opendistroforelasticsearch.sql.domain.Condition; -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Condition; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/multi/ESMultiQueryActionFactory.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/multi/ESMultiQueryActionFactory.java similarity index 83% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/multi/ESMultiQueryActionFactory.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/multi/ESMultiQueryActionFactory.java index a29e8fc1c0..a6d054433f 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/multi/ESMultiQueryActionFactory.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/multi/ESMultiQueryActionFactory.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.multi; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.multi; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.query.QueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.QueryAction; import org.elasticsearch.client.Client; /** diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/multi/MultiQueryAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/multi/MultiQueryAction.java similarity index 86% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/multi/MultiQueryAction.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/multi/MultiQueryAction.java index 401ccc8805..e1bfe48730 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/multi/MultiQueryAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/multi/MultiQueryAction.java @@ -13,14 +13,14 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.multi; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.multi; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.query.DefaultQueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.QueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.SqlElasticRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.DefaultQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.QueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.SqlElasticRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.client.Client; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/multi/MultiQueryRequestBuilder.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/multi/MultiQueryRequestBuilder.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/multi/MultiQueryRequestBuilder.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/multi/MultiQueryRequestBuilder.java index 28a0841d76..8cc42fc023 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/multi/MultiQueryRequestBuilder.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/multi/MultiQueryRequestBuilder.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.multi; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.multi; import com.alibaba.druid.sql.ast.statement.SQLUnionOperator; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.query.SqlElasticRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.SqlElasticRequestBuilder; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.ActionResponse; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/multi/MultiQuerySelect.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/multi/MultiQuerySelect.java similarity index 90% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/multi/MultiQuerySelect.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/multi/MultiQuerySelect.java index c4f6eb3cdc..21cc2f31ab 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/multi/MultiQuerySelect.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/multi/MultiQuerySelect.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.multi; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.multi; import com.alibaba.druid.sql.ast.statement.SQLUnionOperator; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; /** * Created by Eliran on 19/8/2016. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/HashJoinQueryPlanRequestBuilder.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/HashJoinQueryPlanRequestBuilder.java similarity index 81% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/HashJoinQueryPlanRequestBuilder.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/HashJoinQueryPlanRequestBuilder.java index e7dd282cde..440d35f259 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/HashJoinQueryPlanRequestBuilder.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/HashJoinQueryPlanRequestBuilder.java @@ -13,13 +13,13 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner; -import com.amazon.opendistroforelasticsearch.sql.query.join.HashJoinElasticRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.Config; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.QueryParams; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.QueryPlanner; -import com.amazon.opendistroforelasticsearch.sql.request.SqlRequest; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.HashJoinElasticRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.Config; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.QueryParams; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.QueryPlanner; +import com.amazon.opendistroforelasticsearch.sql.legacy.request.SqlRequest; import org.elasticsearch.client.Client; /** diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/converter/SQLAggregationParser.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/converter/SQLAggregationParser.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/converter/SQLAggregationParser.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/converter/SQLAggregationParser.java index c4c5fa44f6..15f78f6756 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/converter/SQLAggregationParser.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/converter/SQLAggregationParser.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.converter; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.converter; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLAggregateExpr; @@ -25,10 +25,10 @@ import com.alibaba.druid.sql.dialect.mysql.ast.expr.MySqlSelectGroupByExpr; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock; import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitorAdapter; -import com.amazon.opendistroforelasticsearch.sql.domain.ColumnTypeProvider; -import com.amazon.opendistroforelasticsearch.sql.expression.core.Expression; -import com.amazon.opendistroforelasticsearch.sql.expression.core.ExpressionFactory; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.ColumnNode; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.ColumnTypeProvider; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.Expression; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.ExpressionFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.ColumnNode; import com.google.common.base.Strings; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/converter/SQLExprToExpressionConverter.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/converter/SQLExprToExpressionConverter.java similarity index 89% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/converter/SQLExprToExpressionConverter.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/converter/SQLExprToExpressionConverter.java index 96b8d5f40a..113f076418 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/converter/SQLExprToExpressionConverter.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/converter/SQLExprToExpressionConverter.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.converter; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.converter; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; @@ -21,10 +21,10 @@ import com.alibaba.druid.sql.ast.expr.SQLCastExpr; import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr; import com.alibaba.druid.sql.ast.expr.SQLValuableExpr; -import com.amazon.opendistroforelasticsearch.sql.expression.core.Expression; -import com.amazon.opendistroforelasticsearch.sql.expression.core.ExpressionFactory; -import com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation; -import com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.Expression; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.ExpressionFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueFactory; import com.google.common.collect.ImmutableMap; import lombok.RequiredArgsConstructor; @@ -34,8 +34,8 @@ import java.util.function.Function; import java.util.stream.Collectors; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.ExpressionFactory.cast; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.ExpressionFactory.literal; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.ExpressionFactory.cast; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.ExpressionFactory.literal; /** * The definition of {@link SQLExpr} to {@link Expression} converter. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/converter/SQLToOperatorConverter.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/converter/SQLToOperatorConverter.java similarity index 74% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/converter/SQLToOperatorConverter.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/converter/SQLToOperatorConverter.java index 4312e2bc09..8a922198d6 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/converter/SQLToOperatorConverter.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/converter/SQLToOperatorConverter.java @@ -13,19 +13,19 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.converter; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.converter; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock; import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitorAdapter; -import com.amazon.opendistroforelasticsearch.sql.domain.ColumnTypeProvider; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.expression.domain.BindingTuple; -import com.amazon.opendistroforelasticsearch.sql.parser.SqlParser; -import com.amazon.opendistroforelasticsearch.sql.query.AggregationQueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.ColumnNode; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.PhysicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.project.PhysicalProject; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.scroll.PhysicalScroll; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.ColumnTypeProvider; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.domain.BindingTuple; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.SqlParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.AggregationQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.ColumnNode; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.PhysicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.project.PhysicalProject; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.scroll.PhysicalScroll; import lombok.Getter; import lombok.SneakyThrows; import org.apache.logging.log4j.LogManager; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/core/BindingTupleQueryPlanner.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/core/BindingTupleQueryPlanner.java similarity index 82% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/core/BindingTupleQueryPlanner.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/core/BindingTupleQueryPlanner.java index c922feb747..43c3c7adf7 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/core/BindingTupleQueryPlanner.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/core/BindingTupleQueryPlanner.java @@ -13,14 +13,14 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.core; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; -import com.amazon.opendistroforelasticsearch.sql.domain.ColumnTypeProvider; -import com.amazon.opendistroforelasticsearch.sql.expression.domain.BindingTuple; -import com.amazon.opendistroforelasticsearch.sql.query.planner.converter.SQLToOperatorConverter; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.PhysicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.scroll.PhysicalScroll; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.ColumnTypeProvider; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.domain.BindingTuple; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.converter.SQLToOperatorConverter; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.PhysicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.scroll.PhysicalScroll; import lombok.Getter; import org.elasticsearch.client.Client; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/core/ColumnNode.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/core/ColumnNode.java similarity index 70% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/core/ColumnNode.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/core/ColumnNode.java index 4e5755aab1..ba053591d7 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/core/ColumnNode.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/core/ColumnNode.java @@ -13,10 +13,11 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.core; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core; -import com.amazon.opendistroforelasticsearch.sql.executor.format.Schema; -import com.amazon.opendistroforelasticsearch.sql.expression.core.Expression; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.Schema; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.Expression; +import com.google.common.base.Strings; import lombok.Builder; import lombok.Getter; import lombok.Setter; @@ -34,4 +35,8 @@ public class ColumnNode { private String alias; private Schema.Type type; private Expression expr; + + public String columnName() { + return Strings.isNullOrEmpty(alias) ? name : alias; + } } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/core/Config.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/core/Config.java similarity index 91% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/core/Config.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/core/Config.java index 52760c1e36..b8fdaca95c 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/core/Config.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/core/Config.java @@ -13,11 +13,11 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.core; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core; -import com.amazon.opendistroforelasticsearch.sql.query.planner.resource.blocksize.AdaptiveBlockSize; -import com.amazon.opendistroforelasticsearch.sql.query.planner.resource.blocksize.BlockSize; -import com.amazon.opendistroforelasticsearch.sql.query.planner.resource.blocksize.BlockSize.FixedBlockSize; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.resource.blocksize.AdaptiveBlockSize; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.resource.blocksize.BlockSize; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.resource.blocksize.BlockSize.FixedBlockSize; /** * Query planner configuration diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/core/ExecuteParams.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/core/ExecuteParams.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/core/ExecuteParams.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/core/ExecuteParams.java index cc17a798a0..505257aa0e 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/core/ExecuteParams.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/core/ExecuteParams.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.core; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core; import java.util.EnumMap; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/core/Plan.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/core/Plan.java similarity index 84% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/core/Plan.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/core/Plan.java index c57f7cfc19..7795ef0216 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/core/Plan.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/core/Plan.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.core; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.PlanNode.Visitor; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.PlanNode.Visitor; /** * Query plan diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/core/PlanNode.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/core/PlanNode.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/core/PlanNode.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/core/PlanNode.java index 582b8199e0..6b853b67c0 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/core/PlanNode.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/core/PlanNode.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.core; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core; /** * Abstract plan node in query plan. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/core/QueryParams.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/core/QueryParams.java similarity index 91% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/core/QueryParams.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/core/QueryParams.java index 634d64b9de..19f21a8dc0 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/core/QueryParams.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/core/QueryParams.java @@ -13,11 +13,11 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.core; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core; import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.query.join.TableInJoinRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.TableInJoinRequestBuilder; import java.util.List; import java.util.Map; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/core/QueryPlanner.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/core/QueryPlanner.java similarity index 71% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/core/QueryPlanner.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/core/QueryPlanner.java index 6981d7243d..7f850a108d 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/core/QueryPlanner.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/core/QueryPlanner.java @@ -13,23 +13,23 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.core; - -import com.amazon.opendistroforelasticsearch.sql.executor.join.MetaSearchResult; -import com.amazon.opendistroforelasticsearch.sql.query.planner.explain.Explanation; -import com.amazon.opendistroforelasticsearch.sql.query.planner.explain.JsonExplanationFormat; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.LogicalPlan; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.PhysicalPlan; -import com.amazon.opendistroforelasticsearch.sql.query.planner.resource.ResourceManager; -import com.amazon.opendistroforelasticsearch.sql.query.planner.resource.Stats; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core; + +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.join.MetaSearchResult; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.explain.Explanation; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.explain.JsonExplanationFormat; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.PhysicalPlan; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.resource.ResourceManager; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.resource.Stats; import org.elasticsearch.client.Client; import org.elasticsearch.search.SearchHit; import java.util.List; -import static com.amazon.opendistroforelasticsearch.sql.query.planner.core.ExecuteParams.ExecuteParamType.CLIENT; -import static com.amazon.opendistroforelasticsearch.sql.query.planner.core.ExecuteParams.ExecuteParamType.RESOURCE_MANAGER; -import static com.amazon.opendistroforelasticsearch.sql.query.planner.core.ExecuteParams.ExecuteParamType.TIMEOUT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.ExecuteParams.ExecuteParamType.CLIENT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.ExecuteParams.ExecuteParamType.RESOURCE_MANAGER; +import static com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.ExecuteParams.ExecuteParamType.TIMEOUT; /** * Query planner that driver the logical planning, physical planning, execute and explain. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/explain/Explanation.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/explain/Explanation.java similarity index 82% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/explain/Explanation.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/explain/Explanation.java index 56162ccbd5..4d946aee5f 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/explain/Explanation.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/explain/Explanation.java @@ -13,14 +13,14 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.explain; - -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.Plan; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.PlanNode; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.PlanNode.Visitor; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.LogicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Group; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.PhysicalOperator; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.explain; + +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.Plan; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.PlanNode; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.PlanNode.Visitor; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.LogicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Group; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.PhysicalOperator; import com.google.common.collect.ImmutableMap; /** diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/explain/ExplanationFormat.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/explain/ExplanationFormat.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/explain/ExplanationFormat.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/explain/ExplanationFormat.java index 5a8148b800..641436f1f9 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/explain/ExplanationFormat.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/explain/ExplanationFormat.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.explain; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.explain; import java.util.Map; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/explain/JsonExplanationFormat.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/explain/JsonExplanationFormat.java similarity index 97% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/explain/JsonExplanationFormat.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/explain/JsonExplanationFormat.java index fed5af9268..8f6ad876b6 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/explain/JsonExplanationFormat.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/explain/JsonExplanationFormat.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.explain; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.explain; import org.json.JSONArray; import org.json.JSONException; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/LogicalOperator.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/LogicalOperator.java similarity index 85% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/LogicalOperator.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/LogicalOperator.java index 3490c3957c..926f79df7a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/LogicalOperator.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/LogicalOperator.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.logical; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.PlanNode; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.PhysicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.PlanNode; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.PhysicalOperator; import org.json.JSONPropertyIgnore; import java.util.Map; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/LogicalPlan.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/LogicalPlan.java similarity index 80% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/LogicalPlan.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/LogicalPlan.java index 142b559897..979203e7d7 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/LogicalPlan.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/LogicalPlan.java @@ -13,24 +13,24 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.logical; - -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.domain.Order; -import com.amazon.opendistroforelasticsearch.sql.query.join.TableInJoinRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.Config; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.Plan; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.QueryParams; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Filter; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Group; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Join; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Join.JoinCondition; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Project; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Sort; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.TableScan; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Top; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.rule.ProjectionPushDown; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.rule.SelectionPushDown; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical; + +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Order; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.TableInJoinRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.Config; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.Plan; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.QueryParams; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Filter; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Group; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Join; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Join.JoinCondition; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Project; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Sort; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.TableScan; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Top; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.rule.ProjectionPushDown; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.rule.SelectionPushDown; import java.util.ArrayList; import java.util.Arrays; @@ -40,7 +40,7 @@ import java.util.function.Function; import java.util.stream.Collectors; -import static com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Project.Visitor; +import static com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Project.Visitor; /** * Logical query plan. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/LogicalPlanVisitor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/LogicalPlanVisitor.java similarity index 77% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/LogicalPlanVisitor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/LogicalPlanVisitor.java index 3e9e2b4ac1..e4f4e88cdd 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/LogicalPlanVisitor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/LogicalPlanVisitor.java @@ -13,17 +13,17 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.logical; - -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.PlanNode; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.PlanNode.Visitor; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Filter; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Group; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Join; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Project; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Sort; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.TableScan; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Top; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical; + +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.PlanNode; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.PlanNode.Visitor; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Filter; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Group; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Join; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Project; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Sort; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.TableScan; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Top; /** * Transformation rule for logical plan tree optimization implemented by standard Visitor pattern. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/node/Filter.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/node/Filter.java similarity index 78% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/node/Filter.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/node/Filter.java index f18d6f6e71..1ce90bfa6d 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/node/Filter.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/node/Filter.java @@ -13,14 +13,14 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.query.join.TableInJoinRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.PlanNode; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.LogicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.PhysicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.TableInJoinRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.PlanNode; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.LogicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.PhysicalOperator; import java.util.HashMap; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/node/Group.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/node/Group.java similarity index 84% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/node/Group.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/node/Group.java index 27c09992d1..91f69994c7 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/node/Group.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/node/Group.java @@ -13,11 +13,11 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.PlanNode; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.LogicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.PhysicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.PlanNode; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.LogicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.PhysicalOperator; import java.util.Map; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/node/Join.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/node/Join.java similarity index 90% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/node/Join.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/node/Join.java index 52785d906c..d7e6286637 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/node/Join.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/node/Join.java @@ -13,13 +13,13 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.PlanNode; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.LogicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.PhysicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.join.BlockHashJoin; -import com.amazon.opendistroforelasticsearch.sql.query.planner.resource.blocksize.BlockSize; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.PlanNode; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.LogicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.PhysicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.join.BlockHashJoin; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.resource.blocksize.BlockSize; import java.util.Map; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/node/Project.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/node/Project.java similarity index 88% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/node/Project.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/node/Project.java index b9e9b3bb2b..777ccb547e 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/node/Project.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/node/Project.java @@ -13,14 +13,14 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node; - -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.PlanNode; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.LogicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.PhysicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.Row; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.estimation.Cost; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node; + +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.PlanNode; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.LogicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.PhysicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.Row; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.estimation.Cost; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; import org.apache.logging.log4j.LogManager; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/node/Sort.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/node/Sort.java similarity index 78% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/node/Sort.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/node/Sort.java index 67e0878159..64a98e8425 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/node/Sort.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/node/Sort.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.PlanNode; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.LogicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.PhysicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.sort.QuickSort; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.PlanNode; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.LogicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.PhysicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.sort.QuickSort; import java.util.List; import java.util.Map; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/node/TableScan.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/node/TableScan.java similarity index 76% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/node/TableScan.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/node/TableScan.java index 40e45605d2..bb094a261a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/node/TableScan.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/node/TableScan.java @@ -13,13 +13,13 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node; -import com.amazon.opendistroforelasticsearch.sql.query.join.TableInJoinRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.PlanNode; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.LogicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.PhysicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.scroll.Scroll; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.TableInJoinRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.PlanNode; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.LogicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.PhysicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.scroll.Scroll; import java.util.Map; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/node/Top.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/node/Top.java similarity index 78% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/node/Top.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/node/Top.java index 43d955235f..e4c3455c93 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/node/Top.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/node/Top.java @@ -13,13 +13,13 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.PlanNode; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.LogicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.PhysicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.Row; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.estimation.Cost; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.PlanNode; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.LogicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.PhysicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.Row; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.estimation.Cost; import java.util.Map; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/rule/ProjectionPushDown.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/rule/ProjectionPushDown.java similarity index 80% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/rule/ProjectionPushDown.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/rule/ProjectionPushDown.java index bfd0c64c9f..4613fe0f6c 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/rule/ProjectionPushDown.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/rule/ProjectionPushDown.java @@ -13,19 +13,19 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.logical.rule; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.rule; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.LogicalPlanVisitor; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Group; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Join; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Project; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.LogicalPlanVisitor; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Group; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Join; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Project; import java.util.Arrays; import java.util.Collection; import java.util.List; -import static com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Join.JoinCondition; +import static com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Join.JoinCondition; import static java.util.stream.Collectors.toList; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/rule/SelectionPushDown.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/rule/SelectionPushDown.java similarity index 77% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/rule/SelectionPushDown.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/rule/SelectionPushDown.java index 996f346649..ef1de5a403 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/logical/rule/SelectionPushDown.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/logical/rule/SelectionPushDown.java @@ -13,11 +13,11 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.logical.rule; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.rule; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.LogicalPlanVisitor; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Filter; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Group; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.LogicalPlanVisitor; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Filter; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Group; /** * Push down selection (filter) diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/PhysicalOperator.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/PhysicalOperator.java similarity index 80% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/PhysicalOperator.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/PhysicalOperator.java index deee696e7f..813329c0a3 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/PhysicalOperator.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/PhysicalOperator.java @@ -13,11 +13,11 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.physical; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.ExecuteParams; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.PlanNode; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.estimation.Cost; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.ExecuteParams; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.PlanNode; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.estimation.Cost; import java.util.Iterator; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/PhysicalPlan.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/PhysicalPlan.java similarity index 82% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/PhysicalPlan.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/PhysicalPlan.java index a145645ddb..8d7f892dad 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/PhysicalPlan.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/PhysicalPlan.java @@ -13,14 +13,14 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.physical; - -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.ExecuteParams; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.Plan; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.PlanNode.Visitor; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.LogicalPlan; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.estimation.Estimation; -import com.amazon.opendistroforelasticsearch.sql.query.planner.resource.ResourceManager; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical; + +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.ExecuteParams; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.Plan; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.PlanNode.Visitor; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.estimation.Estimation; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.resource.ResourceManager; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.search.SearchHit; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/Row.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/Row.java similarity index 97% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/Row.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/Row.java index b16b47c050..d336f7590e 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/Row.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/Row.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.physical; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical; import java.util.Arrays; import java.util.Map; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/estimation/Cost.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/estimation/Cost.java similarity index 90% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/estimation/Cost.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/estimation/Cost.java index 870b68bc51..1bd614788a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/estimation/Cost.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/estimation/Cost.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.physical.estimation; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.estimation; public class Cost implements Comparable { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/estimation/Estimation.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/estimation/Estimation.java similarity index 77% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/estimation/Estimation.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/estimation/Estimation.java index 3af19d17e5..34ed43ad97 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/estimation/Estimation.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/estimation/Estimation.java @@ -13,13 +13,13 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.physical.estimation; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.estimation; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.PlanNode; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.LogicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.LogicalPlanVisitor; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Group; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.PhysicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.PlanNode; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.LogicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.LogicalPlanVisitor; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Group; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.PhysicalOperator; import java.util.Arrays; import java.util.IdentityHashMap; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/BatchPhysicalOperator.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/BatchPhysicalOperator.java similarity index 84% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/BatchPhysicalOperator.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/BatchPhysicalOperator.java index aeb69c216e..56e8a69770 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/BatchPhysicalOperator.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/BatchPhysicalOperator.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.ExecuteParams; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.PhysicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.Row; -import com.amazon.opendistroforelasticsearch.sql.query.planner.resource.ResourceManager; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.ExecuteParams; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.PhysicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.Row; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.resource.ResourceManager; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -26,7 +26,7 @@ import java.util.Iterator; import java.util.Objects; -import static com.amazon.opendistroforelasticsearch.sql.query.planner.core.ExecuteParams.ExecuteParamType.RESOURCE_MANAGER; +import static com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.ExecuteParams.ExecuteParamType.RESOURCE_MANAGER; /** * Abstraction for physical operators that load large volume of data and generally prefetch for efficiency. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/join/BlockHashJoin.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/join/BlockHashJoin.java similarity index 86% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/join/BlockHashJoin.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/join/BlockHashJoin.java index 5e3dc0017d..5256e0095e 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/join/BlockHashJoin.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/join/BlockHashJoin.java @@ -13,13 +13,13 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.join; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.join; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.ExecuteParams; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.PhysicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.Row; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.estimation.Cost; -import com.amazon.opendistroforelasticsearch.sql.query.planner.resource.blocksize.BlockSize; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.ExecuteParams; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.PhysicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.Row; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.estimation.Cost; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.resource.blocksize.BlockSize; import org.elasticsearch.common.Strings; import org.elasticsearch.index.query.BoolQueryBuilder; @@ -30,7 +30,7 @@ import java.util.Objects; import static com.alibaba.druid.sql.ast.statement.SQLJoinTableSource.JoinType; -import static com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Join.JoinCondition; +import static com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Join.JoinCondition; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.termsQuery; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/join/CombinedRow.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/join/CombinedRow.java similarity index 90% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/join/CombinedRow.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/join/CombinedRow.java index 08e2fdd2ae..f6eee015ca 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/join/CombinedRow.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/join/CombinedRow.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.join; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.join; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.Row; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.Row; import java.util.ArrayList; import java.util.Collection; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/join/DefaultHashTable.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/join/DefaultHashTable.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/join/DefaultHashTable.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/join/DefaultHashTable.java index 377cb9eb2d..8ee67e477e 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/join/DefaultHashTable.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/join/DefaultHashTable.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.join; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.join; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.Row; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.Row.RowKey; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.Row; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.Row.RowKey; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; import org.apache.logging.log4j.LogManager; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/join/HashTable.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/join/HashTable.java similarity index 89% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/join/HashTable.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/join/HashTable.java index 9b096c344f..31376b047c 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/join/HashTable.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/join/HashTable.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.join; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.join; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.Row; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.Row; import java.util.Collection; import java.util.Map; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/join/HashTableGroup.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/join/HashTableGroup.java similarity index 90% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/join/HashTableGroup.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/join/HashTableGroup.java index 6a7794044e..b719fd67f5 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/join/HashTableGroup.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/join/HashTableGroup.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.join; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.join; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.Row; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.Row; import com.google.common.collect.Sets; import java.util.Arrays; @@ -23,7 +23,7 @@ import java.util.Map; import java.util.Set; -import static com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Join.JoinCondition; +import static com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Join.JoinCondition; /** * Hash table group with each hash table per AND join condition. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/join/JoinAlgorithm.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/join/JoinAlgorithm.java similarity index 91% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/join/JoinAlgorithm.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/join/JoinAlgorithm.java index f676657bae..c6827ad07e 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/join/JoinAlgorithm.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/join/JoinAlgorithm.java @@ -13,16 +13,16 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.join; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.join; import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource.JoinType; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.ExecuteParams; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.PlanNode; -import com.amazon.opendistroforelasticsearch.sql.query.planner.logical.node.Join.JoinCondition; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.PhysicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.Row; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.BatchPhysicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.resource.blocksize.BlockSize; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.ExecuteParams; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.PlanNode; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.logical.node.Join.JoinCondition; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.PhysicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.Row; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.BatchPhysicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.resource.blocksize.BlockSize; import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/join/ListHashTable.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/join/ListHashTable.java similarity index 89% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/join/ListHashTable.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/join/ListHashTable.java index f4c1fcc021..086ae8e74a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/join/ListHashTable.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/join/ListHashTable.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.join; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.join; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.Row; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.Row; import java.util.ArrayList; import java.util.Collection; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/project/PhysicalProject.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/project/PhysicalProject.java similarity index 67% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/project/PhysicalProject.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/project/PhysicalProject.java index d4f79d5717..f68022ce7d 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/project/PhysicalProject.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/project/PhysicalProject.java @@ -13,15 +13,15 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.project; - -import com.amazon.opendistroforelasticsearch.sql.expression.domain.BindingTuple; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.ColumnNode; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.PlanNode; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.PhysicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.Row; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.estimation.Cost; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.scroll.BindingTupleRow; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.project; + +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.domain.BindingTuple; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.ColumnNode; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.PlanNode; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.PhysicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.Row; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.estimation.Cost; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.scroll.BindingTupleRow; import lombok.RequiredArgsConstructor; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/scroll/BindingTupleRow.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/scroll/BindingTupleRow.java similarity index 82% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/scroll/BindingTupleRow.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/scroll/BindingTupleRow.java index 8593e3e641..4864d44d8f 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/scroll/BindingTupleRow.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/scroll/BindingTupleRow.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.scroll; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.scroll; -import com.amazon.opendistroforelasticsearch.sql.expression.domain.BindingTuple; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.Row; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.domain.BindingTuple; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.Row; import lombok.RequiredArgsConstructor; import java.util.Map; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/scroll/PhysicalScroll.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/scroll/PhysicalScroll.java similarity index 71% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/scroll/PhysicalScroll.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/scroll/PhysicalScroll.java index 2b63f68850..c3d62675c3 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/scroll/PhysicalScroll.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/scroll/PhysicalScroll.java @@ -13,17 +13,17 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.scroll; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.scroll; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.expression.domain.BindingTuple; -import com.amazon.opendistroforelasticsearch.sql.query.AggregationQueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.QueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.ExecuteParams; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.PlanNode; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.PhysicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.Row; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.estimation.Cost; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.domain.BindingTuple; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.AggregationQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.QueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.ExecuteParams; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.PlanNode; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.PhysicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.Row; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.estimation.Cost; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import org.elasticsearch.action.ActionResponse; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/scroll/Scroll.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/scroll/Scroll.java similarity index 86% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/scroll/Scroll.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/scroll/Scroll.java index 743b42304d..860a9d6519 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/scroll/Scroll.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/scroll/Scroll.java @@ -13,18 +13,18 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.scroll; - -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.query.join.TableInJoinRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.maker.QueryMaker; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.ExecuteParams; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.PlanNode; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.Row; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.estimation.Cost; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.BatchPhysicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.resource.ResourceManager; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.scroll; + +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.TableInJoinRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.maker.QueryMaker; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.ExecuteParams; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.PlanNode; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.Row; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.estimation.Cost; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.BatchPhysicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.resource.ResourceManager; import org.elasticsearch.action.search.ClearScrollResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/scroll/SearchAggregationResponseHelper.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/scroll/SearchAggregationResponseHelper.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/scroll/SearchAggregationResponseHelper.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/scroll/SearchAggregationResponseHelper.java index 1d3bb4bdcf..30ff2d205d 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/scroll/SearchAggregationResponseHelper.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/scroll/SearchAggregationResponseHelper.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.scroll; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.scroll; -import com.amazon.opendistroforelasticsearch.sql.expression.domain.BindingTuple; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.domain.BindingTuple; import com.google.common.annotations.VisibleForTesting; import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.Aggregations; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/scroll/SearchHitRow.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRow.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/scroll/SearchHitRow.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRow.java index 3ff84fd1c3..4592bd45e6 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/scroll/SearchHitRow.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRow.java @@ -13,10 +13,11 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.scroll; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.scroll; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.Row; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.Row; import com.google.common.base.Strings; +import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.text.Text; import org.elasticsearch.search.SearchHit; @@ -150,13 +151,17 @@ private Object getValueOfPath(Object source, String path, boolean isIgnoreFirstD } private SearchHit cloneHit(Row other) { + Map documentFields = new HashMap<>(); + Map metaFields = new HashMap<>(); + SearchHit.splitFieldsByMetadata(hit.getFields(), documentFields, metaFields); SearchHit combined = new SearchHit( hit.docId(), hit.getId() + "|" + (other == NULL ? "0" : ((SearchHitRow) other).hit.getId()), new Text( hit.getType() + "|" + (other == NULL ? null : ((SearchHitRow) other).hit.getType()) ), - hit.getFields() + documentFields, + metaFields ); combined.sourceRef(hit.getSourceRef()); combined.getSourceAsMap().clear(); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/sort/QuickSort.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/sort/QuickSort.java similarity index 82% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/sort/QuickSort.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/sort/QuickSort.java index a105f437a8..6467401d2f 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/physical/node/sort/QuickSort.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/physical/node/sort/QuickSort.java @@ -13,14 +13,14 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.sort; - -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.ExecuteParams; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.PlanNode; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.PhysicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.Row; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.estimation.Cost; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.BatchPhysicalOperator; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.sort; + +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.ExecuteParams; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.PlanNode; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.PhysicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.Row; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.estimation.Cost; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.BatchPhysicalOperator; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/resource/ResourceManager.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/resource/ResourceManager.java similarity index 78% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/resource/ResourceManager.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/resource/ResourceManager.java index 1204f86f8a..90cbc2d577 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/resource/ResourceManager.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/resource/ResourceManager.java @@ -13,13 +13,13 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.resource; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.resource; -import com.amazon.opendistroforelasticsearch.sql.executor.join.MetaSearchResult; -import com.amazon.opendistroforelasticsearch.sql.query.join.BackOffRetryStrategy; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.Config; -import com.amazon.opendistroforelasticsearch.sql.query.planner.resource.monitor.Monitor; -import com.amazon.opendistroforelasticsearch.sql.query.planner.resource.monitor.TotalMemoryMonitor; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.join.MetaSearchResult; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.BackOffRetryStrategy; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.Config; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.resource.monitor.Monitor; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.resource.monitor.TotalMemoryMonitor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/resource/Stats.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/resource/Stats.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/resource/Stats.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/resource/Stats.java index 2cdc717100..1f00e92730 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/resource/Stats.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/resource/Stats.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.resource; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.resource; import org.elasticsearch.client.Client; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/resource/blocksize/AdaptiveBlockSize.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/resource/blocksize/AdaptiveBlockSize.java similarity index 92% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/resource/blocksize/AdaptiveBlockSize.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/resource/blocksize/AdaptiveBlockSize.java index 135514adbf..db059fef93 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/resource/blocksize/AdaptiveBlockSize.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/resource/blocksize/AdaptiveBlockSize.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.resource.blocksize; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.resource.blocksize; /** * Adaptive block size calculator based on resource usage dynamically. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/resource/blocksize/BlockSize.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/resource/blocksize/BlockSize.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/resource/blocksize/BlockSize.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/resource/blocksize/BlockSize.java index c96f046055..f8f81229b2 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/resource/blocksize/BlockSize.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/resource/blocksize/BlockSize.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.resource.blocksize; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.resource.blocksize; /** * Block size calculating logic. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/resource/monitor/Monitor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/resource/monitor/Monitor.java similarity index 90% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/resource/monitor/Monitor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/resource/monitor/Monitor.java index 5e849d4296..6e48e66c00 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/resource/monitor/Monitor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/resource/monitor/Monitor.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.resource.monitor; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.resource.monitor; /** * Interface for different monitor component diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/resource/monitor/TotalMemoryMonitor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/resource/monitor/TotalMemoryMonitor.java similarity index 83% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/resource/monitor/TotalMemoryMonitor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/resource/monitor/TotalMemoryMonitor.java index 4c24e3427b..12c9ec6426 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/planner/resource/monitor/TotalMemoryMonitor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/query/planner/resource/monitor/TotalMemoryMonitor.java @@ -13,11 +13,11 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.query.planner.resource.monitor; +package com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.resource.monitor; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.Config; -import com.amazon.opendistroforelasticsearch.sql.query.planner.resource.Stats; -import com.amazon.opendistroforelasticsearch.sql.query.planner.resource.Stats.MemStats; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.Config; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.resource.Stats; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.resource.Stats.MemStats; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/request/PreparedStatementRequest.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/request/PreparedStatementRequest.java similarity index 98% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/request/PreparedStatementRequest.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/request/PreparedStatementRequest.java index e110b1d1f3..3c328a1899 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/request/PreparedStatementRequest.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/request/PreparedStatementRequest.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.request; +package com.amazon.opendistroforelasticsearch.sql.legacy.request; import org.json.JSONObject; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/request/SqlRequest.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/request/SqlRequest.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/request/SqlRequest.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/request/SqlRequest.java index 7e2e3fce7e..dfeddff7e8 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/request/SqlRequest.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/request/SqlRequest.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.request; +package com.amazon.opendistroforelasticsearch.sql.legacy.request; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/request/SqlRequestFactory.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/request/SqlRequestFactory.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/request/SqlRequestFactory.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/request/SqlRequestFactory.java index d285939cab..3f26c137b5 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/request/SqlRequestFactory.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/request/SqlRequestFactory.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.request; +package com.amazon.opendistroforelasticsearch.sql.legacy.request; -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; import org.elasticsearch.rest.RestRequest; import org.json.JSONArray; import org.json.JSONException; @@ -25,7 +25,7 @@ import java.util.List; import java.util.Optional; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.CURSOR_FETCH_SIZE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.CURSOR_FETCH_SIZE; public class SqlRequestFactory { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/request/SqlRequestParam.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/request/SqlRequestParam.java similarity index 87% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/request/SqlRequestParam.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/request/SqlRequestParam.java index 2367333d11..4d72acf92b 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/request/SqlRequestParam.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/request/SqlRequestParam.java @@ -13,15 +13,15 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.request; +package com.amazon.opendistroforelasticsearch.sql.legacy.request; -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; -import com.amazon.opendistroforelasticsearch.sql.executor.Format; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.Format; import java.util.Map; import java.util.Optional; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.QUERY_RESPONSE_FORMAT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.QUERY_RESPONSE_FORMAT; /** * Utils class for parse the request params. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/RewriteRule.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/RewriteRule.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/RewriteRule.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/RewriteRule.java index afd7f055fc..fb16959209 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/RewriteRule.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/RewriteRule.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/RewriteRuleExecutor.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/RewriteRuleExecutor.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/RewriteRuleExecutor.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/RewriteRuleExecutor.java index 4316325737..2bd1bdc776 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/RewriteRuleExecutor.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/RewriteRuleExecutor.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/alias/Identifier.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/alias/Identifier.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/alias/Identifier.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/alias/Identifier.java index 8eb3ec0b90..9df1f92696 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/alias/Identifier.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/alias/Identifier.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.alias; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.alias; import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/alias/Table.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/alias/Table.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/alias/Table.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/alias/Table.java index 5616d898f4..1f4f2fa025 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/alias/Table.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/alias/Table.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.alias; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.alias; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/alias/TableAliasPrefixRemoveRule.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/alias/TableAliasPrefixRemoveRule.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/alias/TableAliasPrefixRemoveRule.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/alias/TableAliasPrefixRemoveRule.java index 72a491abdd..fd76fff192 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/alias/TableAliasPrefixRemoveRule.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/alias/TableAliasPrefixRemoveRule.java @@ -13,15 +13,15 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.alias; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.alias; import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; import com.alibaba.druid.sql.ast.statement.SQLExprTableSource; import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource; import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitorAdapter; -import com.amazon.opendistroforelasticsearch.sql.rewriter.RewriteRule; -import com.amazon.opendistroforelasticsearch.sql.rewriter.subquery.utils.FindSubQuery; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.RewriteRule; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery.utils.FindSubQuery; import java.util.HashSet; import java.util.Set; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/identifier/AnonymizeSensitiveDataRule.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/identifier/AnonymizeSensitiveDataRule.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/identifier/AnonymizeSensitiveDataRule.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/identifier/AnonymizeSensitiveDataRule.java index 8b1d008777..7c7c933f3f 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/identifier/AnonymizeSensitiveDataRule.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/identifier/AnonymizeSensitiveDataRule.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.identifier; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.identifier; import com.alibaba.druid.sql.ast.expr.SQLBooleanExpr; import com.alibaba.druid.sql.ast.expr.SQLCharExpr; @@ -23,7 +23,7 @@ import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; import com.alibaba.druid.sql.ast.statement.SQLExprTableSource; import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitorAdapter; -import com.amazon.opendistroforelasticsearch.sql.rewriter.RewriteRule; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.RewriteRule; /** * Rewrite rule to anonymize sensitive data in logging queries. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/identifier/UnquoteIdentifierRule.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/identifier/UnquoteIdentifierRule.java similarity index 88% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/identifier/UnquoteIdentifierRule.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/identifier/UnquoteIdentifierRule.java index 64339f6b58..faaebe51bf 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/identifier/UnquoteIdentifierRule.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/identifier/UnquoteIdentifierRule.java @@ -13,17 +13,17 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.identifier; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.identifier; import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; import com.alibaba.druid.sql.ast.statement.SQLExprTableSource; import com.alibaba.druid.sql.ast.statement.SQLSelectItem; import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitorAdapter; -import com.amazon.opendistroforelasticsearch.sql.rewriter.RewriteRule; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.RewriteRule; -import static com.amazon.opendistroforelasticsearch.sql.utils.StringUtils.unquoteFullColumn; -import static com.amazon.opendistroforelasticsearch.sql.utils.StringUtils.unquoteSingleField; +import static com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils.unquoteFullColumn; +import static com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils.unquoteSingleField; /** * Quoted Identifiers Rewriter Rule diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/join/JoinRewriteRule.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/join/JoinRewriteRule.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/join/JoinRewriteRule.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/join/JoinRewriteRule.java index 984aeef7f3..b03b812517 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/join/JoinRewriteRule.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/join/JoinRewriteRule.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.join; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.join; import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; @@ -22,11 +22,11 @@ import com.alibaba.druid.sql.ast.statement.SQLSelectQuery; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock; import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitorAdapter; -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; -import com.amazon.opendistroforelasticsearch.sql.esdomain.mapping.FieldMappings; -import com.amazon.opendistroforelasticsearch.sql.rewriter.RewriteRule; -import com.amazon.opendistroforelasticsearch.sql.rewriter.matchtoterm.VerificationException; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.mapping.FieldMappings; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.RewriteRule; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.matchtoterm.VerificationException; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/matchtoterm/TermFieldRewriter.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/matchtoterm/TermFieldRewriter.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/matchtoterm/TermFieldRewriter.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/matchtoterm/TermFieldRewriter.java index 5e7ee0cbd7..faeb981408 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/matchtoterm/TermFieldRewriter.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/matchtoterm/TermFieldRewriter.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.matchtoterm; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.matchtoterm; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLObject; @@ -32,9 +32,9 @@ import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock; import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitorAdapter; import com.alibaba.druid.sql.parser.ParserException; -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; -import com.amazon.opendistroforelasticsearch.sql.esdomain.mapping.FieldMappings; -import com.amazon.opendistroforelasticsearch.sql.esdomain.mapping.IndexMappings; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.mapping.FieldMappings; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.mapping.IndexMappings; import java.util.ArrayDeque; import java.util.Deque; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/matchtoterm/TermFieldScope.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/matchtoterm/TermFieldScope.java similarity index 91% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/matchtoterm/TermFieldScope.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/matchtoterm/TermFieldScope.java index 1286d77966..362802a14a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/matchtoterm/TermFieldScope.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/matchtoterm/TermFieldScope.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.matchtoterm; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.matchtoterm; -import com.amazon.opendistroforelasticsearch.sql.esdomain.mapping.FieldMappings; -import com.amazon.opendistroforelasticsearch.sql.esdomain.mapping.IndexMappings; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.mapping.FieldMappings; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.mapping.IndexMappings; import org.json.JSONObject; import java.util.HashMap; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/matchtoterm/VerificationException.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/matchtoterm/VerificationException.java similarity index 92% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/matchtoterm/VerificationException.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/matchtoterm/VerificationException.java index 82861cf82b..cc139d766e 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/matchtoterm/VerificationException.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/matchtoterm/VerificationException.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.matchtoterm; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.matchtoterm; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.rest.RestStatus; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/nestedfield/From.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/From.java similarity index 98% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/nestedfield/From.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/From.java index 911c6f4402..f3abff88b3 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/nestedfield/From.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/From.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.nestedfield; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.nestedfield; import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; import com.alibaba.druid.sql.ast.statement.SQLExprTableSource; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/nestedfield/Identifier.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/Identifier.java similarity index 97% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/nestedfield/Identifier.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/Identifier.java index d7c0fc22b0..54cf1d0849 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/nestedfield/Identifier.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/Identifier.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.nestedfield; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.nestedfield; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/nestedfield/NestedFieldProjection.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/NestedFieldProjection.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/nestedfield/NestedFieldProjection.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/NestedFieldProjection.java index 6e91aa3ec1..4ae0951fd0 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/nestedfield/NestedFieldProjection.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/NestedFieldProjection.java @@ -13,11 +13,11 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.nestedfield; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.nestedfield; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.rewriter.matchtoterm.VerificationException; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.matchtoterm.VerificationException; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.index.query.BoolQueryBuilder; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/nestedfield/NestedFieldRewriter.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/NestedFieldRewriter.java similarity index 97% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/nestedfield/NestedFieldRewriter.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/NestedFieldRewriter.java index 1d8cf1ec9f..b1087da6df 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/nestedfield/NestedFieldRewriter.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/NestedFieldRewriter.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.nestedfield; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.nestedfield; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; @@ -23,7 +23,7 @@ import java.util.ArrayDeque; import java.util.Deque; -import static com.amazon.opendistroforelasticsearch.sql.utils.Util.NESTED_JOIN_TYPE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.utils.Util.NESTED_JOIN_TYPE; /** * Visitor to rewrite AST (abstract syntax tree) for nested type fields to support implicit nested() function call. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/nestedfield/SQLClause.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/SQLClause.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/nestedfield/SQLClause.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/SQLClause.java index f1116b3bff..8071496f50 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/nestedfield/SQLClause.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/SQLClause.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.nestedfield; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.nestedfield; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLObject; @@ -27,7 +27,7 @@ import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem; import com.alibaba.druid.sql.dialect.mysql.ast.expr.MySqlSelectGroupByExpr; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock; -import com.amazon.opendistroforelasticsearch.sql.utils.Util; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.Util; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/nestedfield/Scope.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/Scope.java similarity index 97% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/nestedfield/Scope.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/Scope.java index c2958e3c55..8ac858b264 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/nestedfield/Scope.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/Scope.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.nestedfield; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.nestedfield; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/nestedfield/Select.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/Select.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/nestedfield/Select.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/Select.java index 7d61849388..b140009d8c 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/nestedfield/Select.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/Select.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.nestedfield; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.nestedfield; import com.alibaba.druid.sql.ast.expr.SQLAllColumnExpr; import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/nestedfield/Where.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/Where.java similarity index 98% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/nestedfield/Where.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/Where.java index dab973daa9..f764d3d101 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/nestedfield/Where.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/nestedfield/Where.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.nestedfield; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.nestedfield; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; import com.alibaba.druid.sql.ast.expr.SQLCharExpr; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/ordinal/OrdinalRewriterRule.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/ordinal/OrdinalRewriterRule.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/ordinal/OrdinalRewriterRule.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/ordinal/OrdinalRewriterRule.java index 6c92984146..e8b49c8596 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/ordinal/OrdinalRewriterRule.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/ordinal/OrdinalRewriterRule.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.ordinal; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.ordinal; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; @@ -26,9 +26,9 @@ import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock; import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitorAdapter; import com.alibaba.druid.sql.parser.SQLExprParser; -import com.amazon.opendistroforelasticsearch.sql.parser.ElasticSqlExprParser; -import com.amazon.opendistroforelasticsearch.sql.rewriter.RewriteRule; -import com.amazon.opendistroforelasticsearch.sql.rewriter.matchtoterm.VerificationException; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.ElasticSqlExprParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.RewriteRule; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.matchtoterm.VerificationException; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/parent/SQLExprParentSetter.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/parent/SQLExprParentSetter.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/parent/SQLExprParentSetter.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/parent/SQLExprParentSetter.java index 06dee94ca2..6b6cb8a093 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/parent/SQLExprParentSetter.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/parent/SQLExprParentSetter.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.parent; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.parent; import com.alibaba.druid.sql.ast.expr.SQLInListExpr; import com.alibaba.druid.sql.ast.expr.SQLInSubQueryExpr; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/parent/SQLExprParentSetterRule.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/parent/SQLExprParentSetterRule.java similarity index 87% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/parent/SQLExprParentSetterRule.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/parent/SQLExprParentSetterRule.java index ea07bdb19b..d014d484de 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/parent/SQLExprParentSetterRule.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/parent/SQLExprParentSetterRule.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.parent; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.parent; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; -import com.amazon.opendistroforelasticsearch.sql.rewriter.RewriteRule; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.RewriteRule; /** * The {@link RewriteRule} which will apply {@link SQLExprParentSetter} for {@link SQLQueryExpr} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/NestedQueryContext.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/NestedQueryContext.java similarity index 97% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/NestedQueryContext.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/NestedQueryContext.java index 210f5cbfbb..56885e20ee 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/NestedQueryContext.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/NestedQueryContext.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.subquery; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery; import com.alibaba.druid.sql.ast.statement.SQLExprTableSource; import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/RewriterContext.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/RewriterContext.java similarity index 97% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/RewriterContext.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/RewriterContext.java index d06a4a728b..2c2517aed3 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/RewriterContext.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/RewriterContext.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.subquery; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/SubQueryRewriteRule.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/SubQueryRewriteRule.java similarity index 85% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/SubQueryRewriteRule.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/SubQueryRewriteRule.java index 67d1792544..0dc63f9101 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/SubQueryRewriteRule.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/SubQueryRewriteRule.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.subquery; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; -import com.amazon.opendistroforelasticsearch.sql.rewriter.RewriteRule; -import com.amazon.opendistroforelasticsearch.sql.rewriter.subquery.rewriter.SubqueryAliasRewriter; -import com.amazon.opendistroforelasticsearch.sql.rewriter.subquery.utils.FindSubQuery; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.RewriteRule; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery.rewriter.SubqueryAliasRewriter; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery.utils.FindSubQuery; import java.sql.SQLFeatureNotSupportedException; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/SubQueryRewriter.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/SubQueryRewriter.java similarity index 91% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/SubQueryRewriter.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/SubQueryRewriter.java index ae911b87ba..54c1e0d904 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/SubQueryRewriter.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/SubQueryRewriter.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.subquery; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; @@ -23,9 +23,9 @@ import com.alibaba.druid.sql.ast.statement.SQLSelectQuery; import com.alibaba.druid.sql.ast.statement.SQLTableSource; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock; -import com.amazon.opendistroforelasticsearch.sql.rewriter.subquery.rewriter.Rewriter; -import com.amazon.opendistroforelasticsearch.sql.rewriter.subquery.rewriter.RewriterFactory; -import com.amazon.opendistroforelasticsearch.sql.rewriter.subquery.utils.FindSubQuery; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery.rewriter.Rewriter; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery.rewriter.RewriterFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery.utils.FindSubQuery; public class SubQueryRewriter { private final RewriterContext ctx = new RewriterContext(); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/rewriter/InRewriter.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/rewriter/InRewriter.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/rewriter/InRewriter.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/rewriter/InRewriter.java index 82d161598d..de3ff49acb 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/rewriter/InRewriter.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/rewriter/InRewriter.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.subquery.rewriter; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery.rewriter; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; @@ -23,7 +23,7 @@ import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource.JoinType; import com.alibaba.druid.sql.ast.statement.SQLTableSource; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock; -import com.amazon.opendistroforelasticsearch.sql.rewriter.subquery.RewriterContext; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery.RewriterContext; /** * IN Subquery Rewriter. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/rewriter/NestedExistsRewriter.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/rewriter/NestedExistsRewriter.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/rewriter/NestedExistsRewriter.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/rewriter/NestedExistsRewriter.java index 28e1a65e7d..28a155c841 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/rewriter/NestedExistsRewriter.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/rewriter/NestedExistsRewriter.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.subquery.rewriter; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery.rewriter; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; @@ -24,7 +24,7 @@ import com.alibaba.druid.sql.ast.statement.SQLExprTableSource; import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource.JoinType; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock; -import com.amazon.opendistroforelasticsearch.sql.rewriter.subquery.RewriterContext; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery.RewriterContext; /** * Nested EXISTS SQL Rewriter. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/rewriter/Rewriter.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/rewriter/Rewriter.java similarity index 92% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/rewriter/Rewriter.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/rewriter/Rewriter.java index 6d6f2f549a..4a12dc82c4 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/rewriter/Rewriter.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/rewriter/Rewriter.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.subquery.rewriter; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery.rewriter; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOperator; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/rewriter/RewriterFactory.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/rewriter/RewriterFactory.java similarity index 91% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/rewriter/RewriterFactory.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/rewriter/RewriterFactory.java index 3bf05baae5..4b54d45316 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/rewriter/RewriterFactory.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/rewriter/RewriterFactory.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.subquery.rewriter; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery.rewriter; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLExistsExpr; import com.alibaba.druid.sql.ast.expr.SQLInSubQueryExpr; -import com.amazon.opendistroforelasticsearch.sql.rewriter.subquery.RewriterContext; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery.RewriterContext; import com.google.common.collect.ImmutableList; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/rewriter/SubqueryAliasRewriter.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/rewriter/SubqueryAliasRewriter.java similarity index 98% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/rewriter/SubqueryAliasRewriter.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/rewriter/SubqueryAliasRewriter.java index 31c7b80059..35a165f0b5 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/rewriter/SubqueryAliasRewriter.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/rewriter/SubqueryAliasRewriter.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.subquery.rewriter; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery.rewriter; import com.alibaba.druid.sql.ast.expr.SQLAllColumnExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/utils/FindSubQuery.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/utils/FindSubQuery.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/utils/FindSubQuery.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/utils/FindSubQuery.java index dc1b32e227..cb6e9b59d9 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/rewriter/subquery/utils/FindSubQuery.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/subquery/utils/FindSubQuery.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.subquery.utils; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery.utils; import com.alibaba.druid.sql.ast.expr.SQLExistsExpr; import com.alibaba.druid.sql.ast.expr.SQLInSubQueryExpr; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/spatial/BoundingBoxFilterParams.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/spatial/BoundingBoxFilterParams.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/spatial/BoundingBoxFilterParams.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/spatial/BoundingBoxFilterParams.java index cc9bec90ae..df4c5ebbfe 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/spatial/BoundingBoxFilterParams.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/spatial/BoundingBoxFilterParams.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.spatial; +package com.amazon.opendistroforelasticsearch.sql.legacy.spatial; /** * Created by Eliran on 1/8/2015. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/spatial/CellFilterParams.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/spatial/CellFilterParams.java similarity index 95% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/spatial/CellFilterParams.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/spatial/CellFilterParams.java index a3abccdcd2..d0f970c6c3 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/spatial/CellFilterParams.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/spatial/CellFilterParams.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.spatial; +package com.amazon.opendistroforelasticsearch.sql.legacy.spatial; /** * Created by Eliran on 15/8/2015. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/spatial/DistanceFilterParams.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/spatial/DistanceFilterParams.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/spatial/DistanceFilterParams.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/spatial/DistanceFilterParams.java index 7c0ee86572..1c94fad43e 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/spatial/DistanceFilterParams.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/spatial/DistanceFilterParams.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.spatial; +package com.amazon.opendistroforelasticsearch.sql.legacy.spatial; /** * Created by Eliran on 1/8/2015. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/spatial/Point.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/spatial/Point.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/spatial/Point.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/spatial/Point.java index 913917d44d..5f1f2a333a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/spatial/Point.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/spatial/Point.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.spatial; +package com.amazon.opendistroforelasticsearch.sql.legacy.spatial; /** * Created by Eliran on 1/8/2015. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/spatial/PolygonFilterParams.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/spatial/PolygonFilterParams.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/spatial/PolygonFilterParams.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/spatial/PolygonFilterParams.java index e48c6daa23..2b148b832e 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/spatial/PolygonFilterParams.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/spatial/PolygonFilterParams.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.spatial; +package com.amazon.opendistroforelasticsearch.sql.legacy.spatial; import java.util.List; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/spatial/RangeDistanceFilterParams.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/spatial/RangeDistanceFilterParams.java similarity index 94% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/spatial/RangeDistanceFilterParams.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/spatial/RangeDistanceFilterParams.java index 69d8bfb3fd..155e871757 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/spatial/RangeDistanceFilterParams.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/spatial/RangeDistanceFilterParams.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.spatial; +package com.amazon.opendistroforelasticsearch.sql.legacy.spatial; /** * Created by Eliran on 15/8/2015. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/spatial/SpatialParamsFactory.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/spatial/SpatialParamsFactory.java similarity index 98% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/spatial/SpatialParamsFactory.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/spatial/SpatialParamsFactory.java index d4c64726a1..04ec96797b 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/spatial/SpatialParamsFactory.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/spatial/SpatialParamsFactory.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.spatial; +package com.amazon.opendistroforelasticsearch.sql.legacy.spatial; import com.alibaba.druid.sql.ast.SQLExpr; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/spatial/WktToGeoJsonConverter.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/spatial/WktToGeoJsonConverter.java similarity index 98% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/spatial/WktToGeoJsonConverter.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/spatial/WktToGeoJsonConverter.java index aa2babaf0f..ba372dfa79 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/spatial/WktToGeoJsonConverter.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/spatial/WktToGeoJsonConverter.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.spatial; +package com.amazon.opendistroforelasticsearch.sql.legacy.spatial; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; import com.google.common.base.Joiner; import java.util.ArrayList; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/JsonPrettyFormatter.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/JsonPrettyFormatter.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/JsonPrettyFormatter.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/JsonPrettyFormatter.java index 39acf36c62..ec546294f3 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/JsonPrettyFormatter.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/JsonPrettyFormatter.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.utils; +package com.amazon.opendistroforelasticsearch.sql.legacy.utils; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/LogUtils.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/LogUtils.java similarity index 97% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/LogUtils.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/LogUtils.java index c1a499ff2c..9f802b6e0b 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/LogUtils.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/LogUtils.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.utils; +package com.amazon.opendistroforelasticsearch.sql.legacy.utils; import org.apache.logging.log4j.ThreadContext; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/QueryDataAnonymizer.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/QueryDataAnonymizer.java similarity index 88% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/QueryDataAnonymizer.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/QueryDataAnonymizer.java index efba0c88d1..b23798274b 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/QueryDataAnonymizer.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/QueryDataAnonymizer.java @@ -13,15 +13,15 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.utils; +package com.amazon.opendistroforelasticsearch.sql.legacy.utils; import com.alibaba.druid.sql.SQLUtils; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; -import com.amazon.opendistroforelasticsearch.sql.rewriter.identifier.AnonymizeSensitiveDataRule; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.identifier.AnonymizeSensitiveDataRule; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import static com.amazon.opendistroforelasticsearch.sql.utils.Util.toSqlExpr; +import static com.amazon.opendistroforelasticsearch.sql.legacy.utils.Util.toSqlExpr; /** * Utility class to mask sensitive information in incoming SQL queries diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SQLFunctions.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java similarity index 98% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SQLFunctions.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java index aafc154aba..27f0045e64 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SQLFunctions.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/SQLFunctions.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.utils; +package com.amazon.opendistroforelasticsearch.sql.legacy.utils; import com.alibaba.druid.sql.SQLUtils; import com.alibaba.druid.sql.ast.SQLExpr; @@ -26,12 +26,12 @@ import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; import com.alibaba.druid.sql.ast.expr.SQLTextLiteralExpr; import com.alibaba.druid.sql.ast.expr.SQLVariantRefExpr; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.domain.KVValue; -import com.amazon.opendistroforelasticsearch.sql.domain.MethodField; -import com.amazon.opendistroforelasticsearch.sql.domain.ScriptMethodField; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.executor.format.Schema; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.KVValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.MethodField; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.ScriptMethodField; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.Schema; import com.google.common.base.Joiner; import com.google.common.base.Strings; import com.google.common.collect.Lists; @@ -46,8 +46,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.amazon.opendistroforelasticsearch.sql.utils.StringUtils.format; -import static com.amazon.opendistroforelasticsearch.sql.utils.StringUtils.isQuoted; +import static com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils.format; +import static com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils.isQuoted; /** * Created by allwefantasy on 8/19/16. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/StringUtils.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/StringUtils.java similarity index 98% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/StringUtils.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/StringUtils.java index a276821232..c48f0a7113 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/StringUtils.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/StringUtils.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.utils; +package com.amazon.opendistroforelasticsearch.sql.legacy.utils; import com.google.common.base.Strings; import com.google.common.primitives.Doubles; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/Util.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/Util.java similarity index 96% rename from src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/Util.java rename to legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/Util.java index 048b9130dc..7c8bbbdf47 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/Util.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/utils/Util.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.utils; +package com.amazon.opendistroforelasticsearch.sql.legacy.utils; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLObject; @@ -34,10 +34,10 @@ import com.alibaba.druid.sql.parser.ParserException; import com.alibaba.druid.sql.parser.SQLExprParser; import com.alibaba.druid.sql.parser.Token; -import com.amazon.opendistroforelasticsearch.sql.domain.IndexStatement; -import com.amazon.opendistroforelasticsearch.sql.domain.KVValue; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.parser.ElasticSqlExprParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.IndexStatement; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.KVValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.ElasticSqlExprParser; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequestBuilder; import org.elasticsearch.client.Client; diff --git a/src/main/resources/es-plugin.properties b/legacy/src/main/resources/es-plugin.properties similarity index 89% rename from src/main/resources/es-plugin.properties rename to legacy/src/main/resources/es-plugin.properties index 341f2a380c..e1975c6be0 100644 --- a/src/main/resources/es-plugin.properties +++ b/legacy/src/main/resources/es-plugin.properties @@ -13,5 +13,5 @@ # permissions and limitations under the License. # -plugin=com.amazon.opendistroforelasticsearch.sql.plugin.SqlPlug +plugin=com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlPlug version=${project.version} \ No newline at end of file diff --git a/src/main/resources/plugin-descriptor.properties b/legacy/src/main/resources/plugin-descriptor.properties similarity index 100% rename from src/main/resources/plugin-descriptor.properties rename to legacy/src/main/resources/plugin-descriptor.properties diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/SymbolSimilarityTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/SymbolSimilarityTest.java similarity index 97% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/SymbolSimilarityTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/SymbolSimilarityTest.java index 825f2a2317..47eea8e075 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/SymbolSimilarityTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/SymbolSimilarityTest.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/SyntaxAnalysisTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/SyntaxAnalysisTest.java similarity index 96% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/SyntaxAnalysisTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/SyntaxAnalysisTest.java index e5d93a3504..af7fceecd9 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/SyntaxAnalysisTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/SyntaxAnalysisTest.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr; -import com.amazon.opendistroforelasticsearch.sql.antlr.syntax.SyntaxAnalysisException; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.syntax.SyntaxAnalysisException; import org.hamcrest.Matchers; import org.junit.Rule; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerAggregateFunctionTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerAggregateFunctionTest.java similarity index 98% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerAggregateFunctionTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerAggregateFunctionTest.java index 864324bbf4..2ff853e9d3 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerAggregateFunctionTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerAggregateFunctionTest.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic; import org.junit.Ignore; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerBasicTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerBasicTest.java similarity index 93% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerBasicTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerBasicTest.java index ce3a54cbd8..51762c05d5 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerBasicTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerBasicTest.java @@ -13,15 +13,15 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.scope.Namespace; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.scope.SemanticContext; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.scope.Symbol; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESIndex; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.visitor.ESMappingLoader; -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.scope.Namespace; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.scope.SemanticContext; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.scope.Symbol; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESIndex; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.visitor.ESMappingLoader; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -29,18 +29,18 @@ import java.util.Map; import java.util.Optional; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.BOOLEAN; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.DATE; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.DOUBLE; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.GEO_POINT; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.INTEGER; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.KEYWORD; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.LONG; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.OBJECT; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.TEXT; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.UNKNOWN; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESIndex.IndexType.INDEX; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESIndex.IndexType.NESTED_FIELD; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.BOOLEAN; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.DATE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.GEO_POINT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.KEYWORD; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.LONG; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.OBJECT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.TEXT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.UNKNOWN; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESIndex.IndexType.INDEX; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESIndex.IndexType.NESTED_FIELD; import static org.hamcrest.Matchers.aMapWithSize; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.hasEntry; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerConfigTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerConfigTest.java similarity index 89% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerConfigTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerConfigTest.java index fdcacdbffc..45d529f544 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerConfigTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerConfigTest.java @@ -13,11 +13,11 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic; -import com.amazon.opendistroforelasticsearch.sql.antlr.OpenDistroSqlAnalyzer; -import com.amazon.opendistroforelasticsearch.sql.antlr.SqlAnalysisConfig; -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.OpenDistroSqlAnalyzer; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.SqlAnalysisConfig; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerConstantTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerConstantTest.java similarity index 93% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerConstantTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerConstantTest.java index b11ebb95af..b2660a116e 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerConstantTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerConstantTest.java @@ -14,7 +14,7 @@ * */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerESScalarFunctionTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerESScalarFunctionTest.java similarity index 97% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerESScalarFunctionTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerESScalarFunctionTest.java index 8a63a36a82..c43bb9f223 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerESScalarFunctionTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerESScalarFunctionTest.java @@ -13,9 +13,8 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic; -import org.junit.Ignore; import org.junit.Test; /** diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerFieldTypeTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerFieldTypeTest.java similarity index 94% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerFieldTypeTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerFieldTypeTest.java index f01450a55e..6262f0e2ea 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerFieldTypeTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerFieldTypeTest.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic; import org.junit.Before; import org.junit.Test; -import static com.amazon.opendistroforelasticsearch.sql.util.MultipleIndexClusterUtils.mockMultipleIndexEnv; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.MultipleIndexClusterUtils.mockMultipleIndexEnv; public class SemanticAnalyzerFieldTypeTest extends SemanticAnalyzerTestBase { @Before diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerFromClauseTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerFromClauseTest.java similarity index 98% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerFromClauseTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerFromClauseTest.java index 2b18690dfd..63a7744519 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerFromClauseTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerFromClauseTest.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic; import org.junit.Ignore; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerIdentifierTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerIdentifierTest.java similarity index 98% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerIdentifierTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerIdentifierTest.java index b8745b802a..1ec2ff1f8a 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerIdentifierTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerIdentifierTest.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic; import org.junit.Ignore; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerMultiQueryTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerMultiQueryTest.java similarity index 98% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerMultiQueryTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerMultiQueryTest.java index eec3480046..46c9fe7599 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerMultiQueryTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerMultiQueryTest.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic; import org.junit.Ignore; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerOperatorTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerOperatorTest.java similarity index 97% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerOperatorTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerOperatorTest.java index 74c13613cb..e9e32c6856 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerOperatorTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerOperatorTest.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerScalarFunctionTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerScalarFunctionTest.java similarity index 98% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerScalarFunctionTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerScalarFunctionTest.java index d23f14d220..d3d1b74c68 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerScalarFunctionTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerScalarFunctionTest.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic; import org.junit.Ignore; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType; import org.junit.Test; /** diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerSubqueryTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerSubqueryTest.java similarity index 95% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerSubqueryTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerSubqueryTest.java index 73118d36b6..dc4bf26956 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerSubqueryTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerSubqueryTest.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic; -import com.amazon.opendistroforelasticsearch.sql.antlr.visitor.EarlyExitAnalysisException; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.visitor.EarlyExitAnalysisException; import org.junit.Test; /** diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerTestBase.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerTestBase.java similarity index 84% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerTestBase.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerTestBase.java index f8aa01b283..8375db3eeb 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerTestBase.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerTestBase.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic; -import com.amazon.opendistroforelasticsearch.sql.antlr.OpenDistroSqlAnalyzer; -import com.amazon.opendistroforelasticsearch.sql.antlr.SqlAnalysisConfig; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.OpenDistroSqlAnalyzer; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.SqlAnalysisConfig; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; import com.google.common.base.Charsets; import com.google.common.io.Resources; import org.hamcrest.Matchers; @@ -32,7 +32,7 @@ import java.util.Arrays; import java.util.Optional; -import static com.amazon.opendistroforelasticsearch.sql.util.CheckScriptContents.mockLocalClusterState; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.CheckScriptContents.mockLocalClusterState; import static java.util.stream.Collectors.toList; import static org.hamcrest.Matchers.allOf; import static org.junit.Assert.assertEquals; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerTests.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerTests.java similarity index 95% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerTests.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerTests.java index 530db020d5..5c4e8b8f18 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/SemanticAnalyzerTests.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/SemanticAnalyzerTests.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic; import org.junit.runner.RunWith; import org.junit.runners.Suite; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/EnvironmentTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/EnvironmentTest.java similarity index 87% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/EnvironmentTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/EnvironmentTest.java index 410fbfd718..1cc20aa763 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/EnvironmentTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/EnvironmentTest.java @@ -13,21 +13,21 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.scope; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.scope; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESIndex; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESIndex; import org.junit.Assert; import org.junit.Test; import java.util.Map; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.BOOLEAN; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.DATE; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.KEYWORD; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.OBJECT; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.TEXT; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESIndex.IndexType.NESTED_FIELD; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.BOOLEAN; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.DATE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.KEYWORD; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.OBJECT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.TEXT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESIndex.IndexType.NESTED_FIELD; import static org.hamcrest.Matchers.aMapWithSize; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.hasEntry; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/SemanticContextTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/SemanticContextTest.java similarity index 94% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/SemanticContextTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/SemanticContextTest.java index dd2f2beea1..95107fe7b4 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/SemanticContextTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/SemanticContextTest.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.scope; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.scope; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/SymbolTableTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/SymbolTableTest.java similarity index 76% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/SymbolTableTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/SymbolTableTest.java index 98ab8bff85..9688cdf918 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/SymbolTableTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/SymbolTableTest.java @@ -13,23 +13,23 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.scope; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.scope; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.TypeExpression; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESIndex; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.TypeExpression; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESIndex; import org.junit.Assert; import org.junit.Test; import java.util.Map; import java.util.Optional; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.BOOLEAN; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.DATE; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.KEYWORD; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.NUMBER; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.TEXT; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESIndex.IndexType.NESTED_FIELD; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.BOOLEAN; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.DATE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.KEYWORD; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.NUMBER; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.TEXT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESIndex.IndexType.NESTED_FIELD; import static org.hamcrest.Matchers.aMapWithSize; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.hasEntry; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/TypeSupplierTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/TypeSupplierTest.java similarity index 85% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/TypeSupplierTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/TypeSupplierTest.java index 82e2eb7f1e..fd81f86ba6 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/scope/TypeSupplierTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/scope/TypeSupplierTest.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.scope; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.scope; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.SemanticAnalysisException; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.SemanticAnalysisException; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/BaseTypeTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/BaseTypeTest.java similarity index 62% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/BaseTypeTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/BaseTypeTest.java index 493dcc8518..c1a3343348 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/BaseTypeTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/BaseTypeTest.java @@ -13,28 +13,28 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESIndex; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESIndex; import org.junit.Ignore; import org.junit.Test; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.BOOLEAN; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.DATE; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.DOUBLE; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.ES_TYPE; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.FLOAT; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.INTEGER; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.KEYWORD; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.LONG; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.NESTED; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.NUMBER; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.SHORT; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.STRING; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.TEXT; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.UNKNOWN; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESIndex.IndexType.NESTED_FIELD; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.BOOLEAN; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.DATE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.ES_TYPE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.FLOAT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.KEYWORD; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.LONG; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.NESTED; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.NUMBER; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.SHORT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.TEXT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.UNKNOWN; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESIndex.IndexType.NESTED_FIELD; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/GenericTypeTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/GenericTypeTest.java similarity index 61% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/GenericTypeTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/GenericTypeTest.java index b2e32b27c8..91a1cfc7be 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/GenericTypeTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/GenericTypeTest.java @@ -13,18 +13,18 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types; import org.junit.Test; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.DOUBLE; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.INTEGER; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.KEYWORD; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.LONG; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.NUMBER; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.TEXT; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.TYPE_ERROR; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.function.ScalarFunction.LOG; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.KEYWORD; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.LONG; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.NUMBER; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.TEXT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.TYPE_ERROR; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.function.ScalarFunction.LOG; import static java.util.Collections.singletonList; import static org.junit.Assert.assertEquals; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/ProductTypeTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/ProductTypeTest.java similarity index 79% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/ProductTypeTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/ProductTypeTest.java index e1013ee320..9fd371dddb 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/ProductTypeTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/ProductTypeTest.java @@ -13,19 +13,19 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.special.Product; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.special.Product; import org.junit.Assert; import org.junit.Test; import java.util.Arrays; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.BOOLEAN; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.INTEGER; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.KEYWORD; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.NUMBER; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.BOOLEAN; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.KEYWORD; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.NUMBER; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.STRING; import static java.util.Collections.singletonList; /** diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/TypeExpressionTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/TypeExpressionTest.java similarity index 68% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/TypeExpressionTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/TypeExpressionTest.java index 0f39b2c1cd..a51b75872e 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/TypeExpressionTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/semantic/types/TypeExpressionTest.java @@ -13,22 +13,22 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types; import org.junit.Test; import java.util.Arrays; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.BOOLEAN; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.DATE; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.DOUBLE; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.GEO_POINT; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.INTEGER; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.NUMBER; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.STRING; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.TYPE_ERROR; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.UNKNOWN; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.special.Generic.T; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.BOOLEAN; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.DATE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.DOUBLE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.GEO_POINT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.NUMBER; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.TYPE_ERROR; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.UNKNOWN; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.special.Generic.T; import static org.junit.Assert.assertEquals; /** diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/visitor/AntlrSqlParseTreeVisitorTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/visitor/AntlrSqlParseTreeVisitorTest.java similarity index 80% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/visitor/AntlrSqlParseTreeVisitorTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/visitor/AntlrSqlParseTreeVisitorTest.java index db3ec2ec2b..c6e8141ae8 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/visitor/AntlrSqlParseTreeVisitorTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/antlr/visitor/AntlrSqlParseTreeVisitorTest.java @@ -13,15 +13,15 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.antlr.visitor; - -import com.amazon.opendistroforelasticsearch.sql.antlr.OpenDistroSqlAnalyzer; -import com.amazon.opendistroforelasticsearch.sql.antlr.SqlAnalysisConfig; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.scope.SemanticContext; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.Type; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.special.Product; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.visitor.TypeChecker; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlFeatureNotImplementedException; +package com.amazon.opendistroforelasticsearch.sql.legacy.antlr.visitor; + +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.OpenDistroSqlAnalyzer; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.SqlAnalysisConfig; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.scope.SemanticContext; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.Type; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.special.Product; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.visitor.TypeChecker; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlFeatureNotImplementedException; import org.antlr.v4.runtime.tree.ParseTree; import org.junit.Assert; import org.junit.Rule; @@ -30,9 +30,9 @@ import java.util.Arrays; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.DATE; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.INTEGER; -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType.UNKNOWN; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.DATE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType.UNKNOWN; import static java.util.Collections.emptyList; /** diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMappingTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/mapping/FieldMappingTest.java similarity index 93% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMappingTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/mapping/FieldMappingTest.java index a83c0bc0e4..72f55ab0d0 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMappingTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/mapping/FieldMappingTest.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.esdomain.mapping; +package com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.mapping; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.MatcherUtils; import org.hamcrest.Matcher; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMappingsTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/mapping/FieldMappingsTest.java similarity index 91% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMappingsTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/mapping/FieldMappingsTest.java index 10df1dc486..3eb9818888 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMappingsTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/esdomain/mapping/FieldMappingsTest.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.esdomain.mapping; +package com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.mapping; -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; import com.google.common.base.Charsets; import com.google.common.io.Resources; import org.junit.After; @@ -27,7 +27,7 @@ import java.util.HashMap; import java.util.Map; -import static com.amazon.opendistroforelasticsearch.sql.util.CheckScriptContents.mockLocalClusterState; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.CheckScriptContents.mockLocalClusterState; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.aMapWithSize; import static org.hamcrest.Matchers.allOf; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/AsyncRestExecutorTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/AsyncRestExecutorTest.java similarity index 87% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/AsyncRestExecutorTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/AsyncRestExecutorTest.java index db9957fea3..54c62e9356 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/AsyncRestExecutorTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/AsyncRestExecutorTest.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor; -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; -import com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings; -import com.amazon.opendistroforelasticsearch.sql.query.QueryAction; -import com.amazon.opendistroforelasticsearch.sql.request.SqlRequest; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.QueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.request.SqlRequest; import org.elasticsearch.client.Client; import org.elasticsearch.rest.RestChannel; import org.elasticsearch.threadpool.ThreadPool; @@ -26,7 +26,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; -import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.junit.MockitoJUnitRunner; import java.util.Map; import java.util.function.Predicate; @@ -45,7 +45,7 @@ /** * Test AsyncRestExecutor behavior. */ -@RunWith(MockitoJUnitRunner.class) +@RunWith(MockitoJUnitRunner.Silent.class) public class AsyncRestExecutorTest { private static final boolean NON_BLOCKING = false; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/csv/CSVResultTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/csv/CSVResultTest.java similarity index 98% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/csv/CSVResultTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/csv/CSVResultTest.java index 5d83ee6981..be3c0cae26 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/csv/CSVResultTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/csv/CSVResultTest.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.csv; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.csv; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/DateFieldFormatterTest.java similarity index 99% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/DateFieldFormatterTest.java index 42a58e966f..52d555be2d 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DateFieldFormatterTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/executor/format/DateFieldFormatterTest.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.executor.format; +package com.amazon.opendistroforelasticsearch.sql.legacy.executor.format; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; diff --git a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryActionTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryActionTest.java new file mode 100644 index 0000000000..cf3f4b6ff7 --- /dev/null +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryActionTest.java @@ -0,0 +1,79 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy.plugin; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSQLQueryAction.NOT_SUPPORTED_YET; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlAction.EXPLAIN_API_ENDPOINT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; + +import com.amazon.opendistroforelasticsearch.sql.sql.domain.SQLQueryRequest; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.cluster.service.ClusterService; +import org.json.JSONObject; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class RestSQLQueryActionTest { + + @Mock + private ClusterService clusterService; + + @Mock + private NodeClient nodeClient; + + @Test + public void handleQueryThatCanSupport() { + SQLQueryRequest request = new SQLQueryRequest( + new JSONObject("{\"query\": \"SELECT -123\"}"), + "SELECT -123", + QUERY_API_ENDPOINT, + ""); + + RestSQLQueryAction queryAction = new RestSQLQueryAction(clusterService); + assertNotSame(NOT_SUPPORTED_YET, queryAction.prepareRequest(request, nodeClient)); + } + + @Test + public void skipExplainThatNotSupport() { + SQLQueryRequest request = new SQLQueryRequest( + new JSONObject("{\"query\": \"SELECT * FROM test\"}"), + "SELECT * FROM test", + EXPLAIN_API_ENDPOINT, + ""); + + RestSQLQueryAction queryAction = new RestSQLQueryAction(clusterService); + assertSame(NOT_SUPPORTED_YET, queryAction.prepareRequest(request, nodeClient)); + } + + @Test + public void skipQueryThatNotSupport() { + SQLQueryRequest request = new SQLQueryRequest( + new JSONObject("{\"query\": \"SELECT * FROM test\"}"), + "SELECT * FROM test", + QUERY_API_ENDPOINT, + ""); + + RestSQLQueryAction queryAction = new RestSQLQueryAction(clusterService); + assertSame(NOT_SUPPORTED_YET, queryAction.prepareRequest(request, nodeClient)); + } + +} \ No newline at end of file diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/rewriter/alias/IdentifierTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/alias/IdentifierTest.java similarity index 96% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/rewriter/alias/IdentifierTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/alias/IdentifierTest.java index 3ab35f0b49..4e5c7389c8 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/rewriter/alias/IdentifierTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/alias/IdentifierTest.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.alias; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.alias; import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; import org.junit.Assert; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/rewriter/alias/TableAliasPrefixRemoveRuleTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/alias/TableAliasPrefixRemoveRuleTest.java similarity index 97% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/rewriter/alias/TableAliasPrefixRemoveRuleTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/alias/TableAliasPrefixRemoveRuleTest.java index 60305e0b2b..382009cfa8 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/rewriter/alias/TableAliasPrefixRemoveRuleTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/alias/TableAliasPrefixRemoveRuleTest.java @@ -13,11 +13,11 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.alias; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.alias; import com.alibaba.druid.sql.SQLUtils; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; -import com.amazon.opendistroforelasticsearch.sql.util.SqlParserUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.SqlParserUtils; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/rewriter/alias/TableTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/alias/TableTest.java similarity index 95% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/rewriter/alias/TableTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/alias/TableTest.java index e31de448b9..363f9b7a33 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/rewriter/alias/TableTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/rewriter/alias/TableTest.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.rewriter.alias; +package com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.alias; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOperator; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/AggregationOptionTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/AggregationOptionTest.java similarity index 85% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/AggregationOptionTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/AggregationOptionTest.java index 21653b7689..923dac28b3 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/AggregationOptionTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/AggregationOptionTest.java @@ -13,15 +13,15 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest; import com.alibaba.druid.sql.ast.expr.SQLAggregateOption; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.parser.SqlParser; -import com.amazon.opendistroforelasticsearch.sql.util.SqlParserUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.SqlParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.SqlParserUtils; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/DateFormatTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/DateFormatTest.java similarity index 92% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/DateFormatTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/DateFormatTest.java index ce6c76671d..67fcdd5693 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/DateFormatTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/DateFormatTest.java @@ -13,20 +13,20 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; import com.alibaba.druid.sql.parser.ParserException; import com.alibaba.druid.sql.parser.Token; -import com.amazon.opendistroforelasticsearch.sql.domain.Order; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.parser.ElasticSqlExprParser; -import com.amazon.opendistroforelasticsearch.sql.parser.SqlParser; -import com.amazon.opendistroforelasticsearch.sql.query.AggregationQueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.maker.QueryMaker; -import com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Order; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.ElasticSqlExprParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.SqlParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.AggregationQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.maker.QueryMaker; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.MatcherUtils; import org.elasticsearch.client.Client; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.index.query.BoolQueryBuilder; @@ -39,7 +39,7 @@ import java.util.List; -import static com.amazon.opendistroforelasticsearch.sql.util.HasFieldWithValue.hasFieldWithValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.HasFieldWithValue.hasFieldWithValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/DateFunctionsTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/DateFunctionsTest.java similarity index 91% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/DateFunctionsTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/DateFunctionsTest.java index 67c73b88cf..bc6d912c9f 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/DateFunctionsTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/DateFunctionsTest.java @@ -13,19 +13,19 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest; -import com.amazon.opendistroforelasticsearch.sql.parser.ScriptFilter; -import com.amazon.opendistroforelasticsearch.sql.parser.SqlParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.ScriptFilter; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.SqlParser; import org.junit.BeforeClass; import org.junit.Test; import static org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField; import static org.junit.Assert.assertTrue; -import static com.amazon.opendistroforelasticsearch.sql.util.CheckScriptContents.getScriptFieldFromQuery; -import static com.amazon.opendistroforelasticsearch.sql.util.CheckScriptContents.getScriptFilterFromQuery; -import static com.amazon.opendistroforelasticsearch.sql.util.CheckScriptContents.scriptContainsString; -import static com.amazon.opendistroforelasticsearch.sql.util.CheckScriptContents.scriptHasPattern; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.CheckScriptContents.getScriptFieldFromQuery; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.CheckScriptContents.getScriptFilterFromQuery; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.CheckScriptContents.scriptContainsString; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.CheckScriptContents.scriptHasPattern; public class DateFunctionsTest { diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/ESClientTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/ESClientTest.java similarity index 95% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/ESClientTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/ESClientTest.java index bdb611065f..c54d0daf23 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/ESClientTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/ESClientTest.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest; -import com.amazon.opendistroforelasticsearch.sql.esdomain.ESClient; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.ESClient; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.search.MultiSearchRequest; import org.elasticsearch.action.search.MultiSearchResponse; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/ErrorMessageFactoryTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/ErrorMessageFactoryTest.java similarity index 87% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/ErrorMessageFactoryTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/ErrorMessageFactoryTest.java index 32c9958fe6..c10ceaeae8 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/ErrorMessageFactoryTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/ErrorMessageFactoryTest.java @@ -13,11 +13,11 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest; -import com.amazon.opendistroforelasticsearch.sql.executor.format.ElasticsearchErrorMessage; -import com.amazon.opendistroforelasticsearch.sql.executor.format.ErrorMessage; -import com.amazon.opendistroforelasticsearch.sql.executor.format.ErrorMessageFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.ElasticsearchErrorMessage; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.ErrorMessage; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.ErrorMessageFactory; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.rest.RestStatus; import org.junit.Assert; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/FormatTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/FormatTest.java similarity index 89% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/FormatTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/FormatTest.java index d9ba2b666a..e52d4b3365 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/FormatTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/FormatTest.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest; -import com.amazon.opendistroforelasticsearch.sql.executor.Format; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.Format; import org.junit.Test; import java.util.Optional; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/HavingTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/HavingTest.java similarity index 95% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/HavingTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/HavingTest.java index 11712b2c82..5b5d6307a9 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/HavingTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/HavingTest.java @@ -13,16 +13,16 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; import com.alibaba.druid.sql.parser.ParserException; -import com.amazon.opendistroforelasticsearch.sql.domain.MethodField; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.parser.SqlParser; -import com.amazon.opendistroforelasticsearch.sql.query.maker.AggMaker; -import com.amazon.opendistroforelasticsearch.sql.util.SqlParserUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.MethodField; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.SqlParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.maker.AggMaker; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.SqlParserUtils; import org.elasticsearch.script.Script; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; @@ -37,7 +37,7 @@ import java.util.Collection; import java.util.Map; -import static com.amazon.opendistroforelasticsearch.sql.util.HasFieldWithValue.hasFieldWithValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.HasFieldWithValue.hasFieldWithValue; import static java.util.stream.Collectors.toMap; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.allOf; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/JSONRequestTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/JSONRequestTest.java similarity index 95% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/JSONRequestTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/JSONRequestTest.java index c801e4dd55..95c506e0bc 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/JSONRequestTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/JSONRequestTest.java @@ -13,20 +13,20 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest; import com.alibaba.druid.sql.parser.ParserException; -import com.amazon.opendistroforelasticsearch.sql.domain.ColumnTypeProvider; -import com.amazon.opendistroforelasticsearch.sql.domain.QueryActionRequest; -import com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.executor.Format; -import com.amazon.opendistroforelasticsearch.sql.executor.format.Schema; -import com.amazon.opendistroforelasticsearch.sql.query.ESActionFactory; -import com.amazon.opendistroforelasticsearch.sql.query.QueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.SqlElasticRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.request.SqlRequest; -import com.amazon.opendistroforelasticsearch.sql.util.CheckScriptContents; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.ColumnTypeProvider; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.QueryActionRequest; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.TestsConstants; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.Format; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.Schema; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.ESActionFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.QueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.SqlElasticRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.request.SqlRequest; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.CheckScriptContents; import com.google.common.io.Files; import org.elasticsearch.client.Client; import org.json.JSONObject; @@ -36,13 +36,12 @@ import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.Mockito; -import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.junit.MockitoJUnitRunner; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.sql.SQLFeatureNotSupportedException; -import java.util.List; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; @@ -50,7 +49,7 @@ import static org.mockito.Matchers.anyInt; import static org.mockito.Mockito.when; -@RunWith(MockitoJUnitRunner.class) +@RunWith(MockitoJUnitRunner.Silent.class) public class JSONRequestTest { @Mock diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/LocalClusterStateTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/LocalClusterStateTest.java similarity index 87% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/LocalClusterStateTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/LocalClusterStateTest.java index 5528468fd8..60d5ce920c 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/LocalClusterStateTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/LocalClusterStateTest.java @@ -13,27 +13,26 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest; - -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; -import com.amazon.opendistroforelasticsearch.sql.esdomain.mapping.FieldMappings; -import com.amazon.opendistroforelasticsearch.sql.esdomain.mapping.IndexMappings; -import com.amazon.opendistroforelasticsearch.sql.esdomain.mapping.TypeMappings; -import com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants; -import com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest; + +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.mapping.FieldMappings; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.mapping.IndexMappings; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.mapping.TypeMappings; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.TestsConstants; +import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.service.ClusterService; -import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import java.io.IOException; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.QUERY_SLOWLOG; -import static com.amazon.opendistroforelasticsearch.sql.util.CheckScriptContents.mockClusterService; -import static com.amazon.opendistroforelasticsearch.sql.util.CheckScriptContents.mockLocalClusterState; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.QUERY_SLOWLOG; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.CheckScriptContents.mockClusterService; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.CheckScriptContents.mockLocalClusterState; import static java.util.Collections.emptyList; import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; @@ -173,19 +172,19 @@ public void getMappingFromCache() throws IOException { for (int i = 0; i < 10; i++) { LocalClusterState.state().getFieldMappings(new String[]{INDEX_NAME}); } - verify(mockService.state().metaData(), times(1)).findMappings(eq(new String[]{INDEX_NAME}), any(), any()); + verify(mockService.state().metadata(), times(1)).findMappings(eq(new String[]{INDEX_NAME}), any(), any()); // 2.Fire cluster state change event Assert.assertNotNull(listener[0]); ClusterChangedEvent mockEvent = mock(ClusterChangedEvent.class); - when(mockEvent.metaDataChanged()).thenReturn(true); + when(mockEvent.metadataChanged()).thenReturn(true); listener[0].clusterChanged(mockEvent); // 3.Cache should be invalidated and call findMapping another time only for (int i = 0; i < 5; i++) { LocalClusterState.state().getFieldMappings(new String[]{INDEX_NAME}); } - verify(mockService.state().metaData(), times(2)).findMappings(eq(new String[]{INDEX_NAME}), any(), any()); + verify(mockService.state().metadata(), times(2)).findMappings(eq(new String[]{INDEX_NAME}), any(), any()); } @Test diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/MathFunctionsTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/MathFunctionsTest.java similarity index 98% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/MathFunctionsTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/MathFunctionsTest.java index dbb605627e..ed08283ea3 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/MathFunctionsTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/MathFunctionsTest.java @@ -13,11 +13,11 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest; -import com.amazon.opendistroforelasticsearch.sql.parser.ScriptFilter; -import com.amazon.opendistroforelasticsearch.sql.parser.SqlParser; -import com.amazon.opendistroforelasticsearch.sql.util.CheckScriptContents; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.ScriptFilter; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.SqlParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.CheckScriptContents; import org.junit.BeforeClass; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/NestedFieldProjectionTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/NestedFieldProjectionTest.java similarity index 94% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/NestedFieldProjectionTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/NestedFieldProjectionTest.java index 0c9268ddb5..f76b3c2f5b 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/NestedFieldProjectionTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/NestedFieldProjectionTest.java @@ -13,21 +13,21 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; import com.alibaba.druid.sql.parser.ParserException; import com.alibaba.druid.sql.parser.Token; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.rewriter.nestedfield.NestedFieldProjection; -import com.amazon.opendistroforelasticsearch.sql.parser.ElasticSqlExprParser; -import com.amazon.opendistroforelasticsearch.sql.parser.SqlParser; -import com.amazon.opendistroforelasticsearch.sql.query.DefaultQueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.maker.QueryMaker; -import com.amazon.opendistroforelasticsearch.sql.rewriter.nestedfield.NestedFieldRewriter; -import com.amazon.opendistroforelasticsearch.sql.util.HasFieldWithValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.nestedfield.NestedFieldProjection; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.ElasticSqlExprParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.SqlParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.DefaultQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.maker.QueryMaker; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.nestedfield.NestedFieldRewriter; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.HasFieldWithValue; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequestBuilder; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/NestedFieldRewriterTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/NestedFieldRewriterTest.java similarity index 98% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/NestedFieldRewriterTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/NestedFieldRewriterTest.java index e13a991296..03bf82d608 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/NestedFieldRewriterTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/NestedFieldRewriterTest.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLInSubQueryExpr; @@ -27,13 +27,13 @@ import com.alibaba.druid.sql.ast.statement.SQLTableSource; import com.alibaba.druid.sql.ast.statement.SQLUnionQuery; import com.alibaba.druid.sql.dialect.mysql.ast.expr.MySqlSelectGroupByExpr; -import com.amazon.opendistroforelasticsearch.sql.rewriter.nestedfield.NestedFieldRewriter; -import com.amazon.opendistroforelasticsearch.sql.util.SqlParserUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.nestedfield.NestedFieldRewriter; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.SqlParserUtils; import org.junit.Test; import java.util.List; -import static com.amazon.opendistroforelasticsearch.sql.util.SqlParserUtils.parse; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.SqlParserUtils.parse; import static java.util.stream.IntStream.range; import static org.junit.Assert.assertEquals; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/PreparedStatementRequestTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/PreparedStatementRequestTest.java similarity index 96% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/PreparedStatementRequestTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/PreparedStatementRequestTest.java index d3e0d041f9..0c1a88ae88 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/PreparedStatementRequestTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/PreparedStatementRequestTest.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest; -import com.amazon.opendistroforelasticsearch.sql.request.PreparedStatementRequest; +import com.amazon.opendistroforelasticsearch.sql.legacy.request.PreparedStatementRequest; import org.json.JSONObject; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/QueryFunctionsTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/QueryFunctionsTest.java similarity index 95% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/QueryFunctionsTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/QueryFunctionsTest.java index 82db58ac03..b939c8696e 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/QueryFunctionsTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/QueryFunctionsTest.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest; -import com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.query.ESActionFactory; -import com.amazon.opendistroforelasticsearch.sql.util.CheckScriptContents; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.TestsConstants; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.ESActionFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.CheckScriptContents; import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.client.Client; import org.elasticsearch.common.Strings; @@ -31,7 +31,7 @@ import java.sql.SQLFeatureNotSupportedException; -import static com.amazon.opendistroforelasticsearch.sql.util.SqlExplainUtils.explain; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.SqlExplainUtils.explain; import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; import static org.elasticsearch.index.query.QueryBuilders.matchPhraseQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/SqlRequestFactoryTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/SqlRequestFactoryTest.java similarity index 96% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/SqlRequestFactoryTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/SqlRequestFactoryTest.java index 537a5863ac..8bff20f5cc 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/SqlRequestFactoryTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/SqlRequestFactoryTest.java @@ -13,11 +13,11 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest; -import com.amazon.opendistroforelasticsearch.sql.request.PreparedStatementRequest; -import com.amazon.opendistroforelasticsearch.sql.request.SqlRequest; -import com.amazon.opendistroforelasticsearch.sql.request.SqlRequestFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.request.PreparedStatementRequest; +import com.amazon.opendistroforelasticsearch.sql.legacy.request.SqlRequest; +import com.amazon.opendistroforelasticsearch.sql.legacy.request.SqlRequestFactory; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.rest.RestRequest; import org.junit.Assert; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/SqlRequestParamTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/SqlRequestParamTest.java similarity index 84% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/SqlRequestParamTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/SqlRequestParamTest.java index 85845cd0cb..6d6c9f2d52 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/SqlRequestParamTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/SqlRequestParamTest.java @@ -13,20 +13,20 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest; -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; -import com.amazon.opendistroforelasticsearch.sql.executor.Format; -import com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings; -import com.amazon.opendistroforelasticsearch.sql.request.SqlRequestParam; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.Format; +import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings; +import com.amazon.opendistroforelasticsearch.sql.legacy.request.SqlRequestParam; import com.google.common.collect.ImmutableMap; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import static com.amazon.opendistroforelasticsearch.sql.request.SqlRequestParam.QUERY_PARAMS_FORMAT; -import static com.amazon.opendistroforelasticsearch.sql.request.SqlRequestParam.QUERY_PARAMS_PRETTY; +import static com.amazon.opendistroforelasticsearch.sql.legacy.request.SqlRequestParam.QUERY_PARAMS_FORMAT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.request.SqlRequestParam.QUERY_PARAMS_PRETTY; import static java.util.Collections.emptyList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/StringOperatorsTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/StringOperatorsTest.java similarity index 96% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/StringOperatorsTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/StringOperatorsTest.java index 9506469be3..0cad9e71e4 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/StringOperatorsTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/StringOperatorsTest.java @@ -13,11 +13,11 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest; -import com.amazon.opendistroforelasticsearch.sql.parser.ScriptFilter; -import com.amazon.opendistroforelasticsearch.sql.parser.SqlParser; -import com.amazon.opendistroforelasticsearch.sql.util.CheckScriptContents; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.ScriptFilter; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.SqlParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.CheckScriptContents; import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField; import org.junit.BeforeClass; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/WhereWithBoolConditionTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/WhereWithBoolConditionTest.java similarity index 83% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/WhereWithBoolConditionTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/WhereWithBoolConditionTest.java index fd2dfa211e..6d73ee6075 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/WhereWithBoolConditionTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/WhereWithBoolConditionTest.java @@ -13,14 +13,14 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest; -import com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.query.ESActionFactory; -import com.amazon.opendistroforelasticsearch.sql.query.QueryAction; -import com.amazon.opendistroforelasticsearch.sql.util.CheckScriptContents; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.TestsConstants; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.ESActionFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.QueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.CheckScriptContents; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; import com.google.common.io.Files; import org.elasticsearch.client.Client; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/cursor/DefaultCursorTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/cursor/DefaultCursorTest.java similarity index 90% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/cursor/DefaultCursorTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/cursor/DefaultCursorTest.java index 5fd1626a12..da6b60a71e 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/cursor/DefaultCursorTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/cursor/DefaultCursorTest.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.cursor; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.cursor; -import com.amazon.opendistroforelasticsearch.sql.cursor.CursorType; -import com.amazon.opendistroforelasticsearch.sql.cursor.DefaultCursor; +import com.amazon.opendistroforelasticsearch.sql.legacy.cursor.CursorType; +import com.amazon.opendistroforelasticsearch.sql.legacy.cursor.DefaultCursor; import org.junit.Test; import java.util.ArrayList; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/domain/ColumnTypeProviderTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/domain/ColumnTypeProviderTest.java similarity index 73% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/domain/ColumnTypeProviderTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/domain/ColumnTypeProviderTest.java index be4a3087b7..ec74faa368 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/domain/ColumnTypeProviderTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/domain/ColumnTypeProviderTest.java @@ -13,17 +13,17 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.domain; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.domain; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.operator.SetOperator; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.special.Product; -import com.amazon.opendistroforelasticsearch.sql.domain.ColumnTypeProvider; -import com.amazon.opendistroforelasticsearch.sql.executor.format.Schema; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.operator.SetOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.special.Product; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.ColumnTypeProvider; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.Schema; import com.google.common.collect.ImmutableList; import org.junit.Test; -import static com.amazon.opendistroforelasticsearch.sql.domain.ColumnTypeProvider.COLUMN_DEFAULT_TYPE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.domain.ColumnTypeProvider.COLUMN_DEFAULT_TYPE; import static org.junit.Assert.assertEquals; public class ColumnTypeProviderTest { diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/executor/DeleteResultSetTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/executor/DeleteResultSetTest.java similarity index 87% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/executor/DeleteResultSetTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/executor/DeleteResultSetTest.java index d23f8b04c8..1ed630713b 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/executor/DeleteResultSetTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/executor/DeleteResultSetTest.java @@ -13,13 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.executor; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.executor; -import com.amazon.opendistroforelasticsearch.sql.domain.Delete; -import com.amazon.opendistroforelasticsearch.sql.executor.format.DataRows; -import com.amazon.opendistroforelasticsearch.sql.executor.format.DeleteResultSet; -import com.amazon.opendistroforelasticsearch.sql.executor.format.Schema; -import org.apache.lucene.search.TotalHits; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Delete; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.DataRows; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.DeleteResultSet; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.Schema; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.xcontent.DeprecationHandler; diff --git a/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/executor/format/BindingTupleResultSetTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/executor/format/BindingTupleResultSetTest.java new file mode 100644 index 0000000000..fef4c22870 --- /dev/null +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/executor/format/BindingTupleResultSetTest.java @@ -0,0 +1,89 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.executor.format; + +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.BindingTupleResultSet; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.DataRows; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.Schema; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.domain.BindingTuple; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.ColumnNode; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.hamcrest.Matcher; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.MatcherUtils.featureValueOf; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.hasEntry; + +public class BindingTupleResultSetTest { + + @Test + public void buildDataRowsFromBindingTupleShouldPass() { + assertThat(row( + Arrays.asList( + ColumnNode.builder().name("age").type(Schema.Type.INTEGER).build(), + ColumnNode.builder().name("gender").type(Schema.Type.TEXT).build()), + Arrays.asList(BindingTuple.from(ImmutableMap.of("age", 31, "gender", "m")), + BindingTuple.from(ImmutableMap.of("age", 31, "gender", "f")), + BindingTuple.from(ImmutableMap.of("age", 39, "gender", "m")), + BindingTuple.from(ImmutableMap.of("age", 39, "gender", "f")))), + containsInAnyOrder(rowContents(allOf(hasEntry("age", 31), hasEntry("gender", (Object) "m"))), + rowContents(allOf(hasEntry("age", 31), hasEntry("gender", (Object) "f"))), + rowContents(allOf(hasEntry("age", 39), hasEntry("gender", (Object) "m"))), + rowContents(allOf(hasEntry("age", 39), hasEntry("gender", (Object) "f"))))); + } + + @Test + public void buildDataRowsFromBindingTupleIncludeLongValueShouldPass() { + assertThat(row( + Arrays.asList( + ColumnNode.builder().name("longValue").type(Schema.Type.LONG).build(), + ColumnNode.builder().name("gender").type(Schema.Type.TEXT).build()), + Arrays.asList( + BindingTuple.from(ImmutableMap.of("longValue", Long.MAX_VALUE, "gender", "m")), + BindingTuple.from(ImmutableMap.of("longValue", Long.MIN_VALUE, "gender", "f")))), + containsInAnyOrder( + rowContents(allOf(hasEntry("longValue", Long.MAX_VALUE), hasEntry("gender", (Object) "m"))), + rowContents(allOf(hasEntry("longValue", Long.MIN_VALUE), hasEntry("gender", (Object) "f"))))); + } + + @Test + public void buildDataRowsFromBindingTupleIncludeDateShouldPass() { + assertThat(row( + Arrays.asList( + ColumnNode.builder().alias("dateValue").type(Schema.Type.DATE).build(), + ColumnNode.builder().alias("gender").type(Schema.Type.TEXT).build()), + Arrays.asList( + BindingTuple.from(ImmutableMap.of("dateValue", 1529712000000L, "gender", "m")))), + containsInAnyOrder( + rowContents(allOf(hasEntry("dateValue", "2018-06-23 00:00:00.000"), hasEntry("gender", (Object) "m"))))); + } + + private static Matcher rowContents(Matcher> matcher) { + return featureValueOf("DataRows.Row", matcher, DataRows.Row::getContents); + } + + private List row(List columnNodes, List bindingTupleList) { + return ImmutableList.copyOf(BindingTupleResultSet.buildDataRows(columnNodes, bindingTupleList).iterator()); + } +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/executor/format/CSVResultsExtractorTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/executor/format/CSVResultsExtractorTest.java similarity index 82% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/executor/format/CSVResultsExtractorTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/executor/format/CSVResultsExtractorTest.java index 2f46945453..7d794d44b6 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/executor/format/CSVResultsExtractorTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/executor/format/CSVResultsExtractorTest.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.executor.format; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.executor.format; -import com.amazon.opendistroforelasticsearch.sql.executor.csv.CSVResult; -import com.amazon.opendistroforelasticsearch.sql.executor.csv.CSVResultsExtractor; -import com.amazon.opendistroforelasticsearch.sql.executor.csv.CsvExtractorException; -import com.amazon.opendistroforelasticsearch.sql.expression.domain.BindingTuple; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.csv.CSVResult; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.csv.CSVResultsExtractor; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.csv.CsvExtractorException; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.domain.BindingTuple; import com.google.common.collect.ImmutableMap; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/executor/join/ElasticUtilsTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/executor/join/ElasticUtilsTest.java similarity index 89% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/executor/join/ElasticUtilsTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/executor/join/ElasticUtilsTest.java index 1bda31e0ce..71d0ab462c 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/executor/join/ElasticUtilsTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/executor/join/ElasticUtilsTest.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.executor.join; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.executor.join; -import com.amazon.opendistroforelasticsearch.sql.executor.join.ElasticUtils; -import com.amazon.opendistroforelasticsearch.sql.executor.join.MetaSearchResult; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.join.ElasticUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.join.MetaSearchResult; import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.TotalHits.Relation; import org.elasticsearch.search.SearchHit; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/expression/core/BinaryExpressionTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/expression/core/BinaryExpressionTest.java similarity index 83% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/expression/core/BinaryExpressionTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/expression/core/BinaryExpressionTest.java index 27b9e29e29..a014cadb59 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/expression/core/BinaryExpressionTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/expression/core/BinaryExpressionTest.java @@ -13,19 +13,19 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.expression.core; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.expression.core; -import com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.mockito.runners.MockitoJUnitRunner; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.ExpressionFactory.literal; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.ExpressionFactory.ref; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueFactory.integerValue; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueFactory.stringValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.ExpressionFactory.literal; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.ExpressionFactory.ref; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueFactory.integerValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueFactory.stringValue; import static org.junit.Assert.assertEquals; @RunWith(MockitoJUnitRunner.class) diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/expression/core/CompoundExpressionTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/expression/core/CompoundExpressionTest.java similarity index 67% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/expression/core/CompoundExpressionTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/expression/core/CompoundExpressionTest.java index 76701cb5fc..46da129dfc 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/expression/core/CompoundExpressionTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/expression/core/CompoundExpressionTest.java @@ -13,15 +13,15 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.expression.core; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.expression.core; -import com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation; import org.junit.Test; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.ExpressionFactory.literal; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueFactory.doubleValue; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueFactory.integerValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.ExpressionFactory.literal; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueFactory.doubleValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueFactory.integerValue; import static org.junit.Assert.assertEquals; public class CompoundExpressionTest extends ExpressionTest { diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/expression/core/ExpressionTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/expression/core/ExpressionTest.java similarity index 77% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/expression/core/ExpressionTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/expression/core/ExpressionTest.java index f75777d81c..0714a4ac91 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/expression/core/ExpressionTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/expression/core/ExpressionTest.java @@ -13,17 +13,17 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.expression.core; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.expression.core; -import com.amazon.opendistroforelasticsearch.sql.expression.core.Expression; -import com.amazon.opendistroforelasticsearch.sql.expression.core.ExpressionFactory; -import com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation; -import com.amazon.opendistroforelasticsearch.sql.expression.domain.BindingTuple; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.Expression; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.ExpressionFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.domain.BindingTuple; import org.json.JSONObject; import java.util.Arrays; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueUtils.getNumberValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueUtils.getNumberValue; public class ExpressionTest { diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/expression/core/RefExpressionTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/expression/core/RefExpressionTest.java similarity index 63% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/expression/core/RefExpressionTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/expression/core/RefExpressionTest.java index 0deccb298a..ea0ce784f6 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/expression/core/RefExpressionTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/expression/core/RefExpressionTest.java @@ -13,20 +13,20 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.expression.core; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.expression.core; import org.junit.Test; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.ExpressionFactory.ref; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueFactory.doubleValue; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueFactory.integerValue; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueFactory.stringValue; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueUtils.getBooleanValue; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueUtils.getCollectionValue; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueUtils.getDoubleValue; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueUtils.getIntegerValue; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueUtils.getStringValue; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueUtils.getTupleValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.ExpressionFactory.ref; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueFactory.doubleValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueFactory.integerValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueFactory.stringValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueUtils.getBooleanValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueUtils.getCollectionValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueUtils.getDoubleValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueUtils.getIntegerValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueUtils.getStringValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueUtils.getTupleValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.contains; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/expression/core/UnaryExpressionTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/expression/core/UnaryExpressionTest.java similarity index 88% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/expression/core/UnaryExpressionTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/expression/core/UnaryExpressionTest.java index ae898d05b7..7eebcfbd16 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/expression/core/UnaryExpressionTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/expression/core/UnaryExpressionTest.java @@ -13,18 +13,18 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.expression.core; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.expression.core; -import com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.mockito.runners.MockitoJUnitRunner; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.ExpressionFactory.literal; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueFactory.doubleValue; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueFactory.stringValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.ExpressionFactory.literal; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueFactory.doubleValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueFactory.stringValue; import static org.junit.Assert.assertEquals; @RunWith(MockitoJUnitRunner.class) diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/expression/model/ExprValueUtilsTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/expression/model/ExprValueUtilsTest.java similarity index 83% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/expression/model/ExprValueUtilsTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/expression/model/ExprValueUtilsTest.java index d350e8281c..9c986cf86f 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/expression/model/ExprValueUtilsTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/expression/model/ExprValueUtilsTest.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.expression.model; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.expression.model; -import com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueFactory; -import com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueUtils; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -46,6 +46,11 @@ public void getIntegerWithDoubleExprValueShouldPass() { assertThat(ExprValueUtils.getIntegerValue(ExprValueFactory.doubleValue(1d)), equalTo(1)); } + @Test + public void getLongValueFromLongExprValueShouldPass() { + assertThat(ExprValueUtils.getLongValue(ExprValueFactory.from(1L)), equalTo(1L)); + } + @Test public void getIntegerValueFromStringExprValueShouldThrowException() { exceptionRule.expect(IllegalStateException.class); diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/metrics/BasicCounterTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/metrics/BasicCounterTest.java similarity index 88% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/metrics/BasicCounterTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/metrics/BasicCounterTest.java index c662216356..b88eec3ff0 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/metrics/BasicCounterTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/metrics/BasicCounterTest.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.metrics; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.metrics; -import com.amazon.opendistroforelasticsearch.sql.metrics.BasicCounter; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.BasicCounter; import org.junit.Test; import static org.hamcrest.MatcherAssert.assertThat; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/metrics/GaugeMetricTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/metrics/GaugeMetricTest.java similarity index 87% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/metrics/GaugeMetricTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/metrics/GaugeMetricTest.java index 0ae7779c3e..b2886c1e5a 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/metrics/GaugeMetricTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/metrics/GaugeMetricTest.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.metrics; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.metrics; -import com.amazon.opendistroforelasticsearch.sql.metrics.GaugeMetric; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.GaugeMetric; import org.junit.Test; import static org.hamcrest.MatcherAssert.assertThat; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/metrics/MetricsTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/metrics/MetricsTest.java similarity index 88% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/metrics/MetricsTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/metrics/MetricsTest.java index f66be1977b..817f901cb7 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/metrics/MetricsTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/metrics/MetricsTest.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.metrics; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.metrics; -import com.amazon.opendistroforelasticsearch.sql.metrics.BasicCounter; -import com.amazon.opendistroforelasticsearch.sql.metrics.Metric; -import com.amazon.opendistroforelasticsearch.sql.metrics.Metrics; -import com.amazon.opendistroforelasticsearch.sql.metrics.NumericMetric; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.BasicCounter; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.Metric; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.Metrics; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.NumericMetric; import org.json.JSONObject; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/metrics/NumericMetricTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/metrics/NumericMetricTest.java similarity index 84% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/metrics/NumericMetricTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/metrics/NumericMetricTest.java index 3c0737f331..b095e2d4bf 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/metrics/NumericMetricTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/metrics/NumericMetricTest.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.metrics; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.metrics; -import com.amazon.opendistroforelasticsearch.sql.metrics.BasicCounter; -import com.amazon.opendistroforelasticsearch.sql.metrics.NumericMetric; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.BasicCounter; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.NumericMetric; import org.junit.Test; import static org.hamcrest.MatcherAssert.assertThat; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/metrics/RollingCounterTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/metrics/RollingCounterTest.java similarity index 94% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/metrics/RollingCounterTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/metrics/RollingCounterTest.java index db0245774a..843a736427 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/metrics/RollingCounterTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/metrics/RollingCounterTest.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.metrics; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.metrics; -import com.amazon.opendistroforelasticsearch.sql.metrics.RollingCounter; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.RollingCounter; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/parser/BucketPathTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/parser/BucketPathTest.java similarity index 89% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/parser/BucketPathTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/parser/BucketPathTest.java index 842d93560f..66248ebc93 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/parser/BucketPathTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/parser/BucketPathTest.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.parser; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.parser; -import com.amazon.opendistroforelasticsearch.sql.domain.bucketpath.BucketPath; -import com.amazon.opendistroforelasticsearch.sql.domain.bucketpath.Path; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.bucketpath.BucketPath; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.bucketpath.Path; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/parser/FieldMakerTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/parser/FieldMakerTest.java similarity index 87% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/parser/FieldMakerTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/parser/FieldMakerTest.java index 6dfe9f86f4..7a6cc9a848 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/parser/FieldMakerTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/parser/FieldMakerTest.java @@ -13,13 +13,13 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.parser; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.parser; import com.alibaba.druid.sql.ast.expr.SQLIntegerExpr; import com.alibaba.druid.sql.ast.expr.SQLNumberExpr; -import com.amazon.opendistroforelasticsearch.sql.domain.MethodField; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.parser.FieldMaker; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.MethodField; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.FieldMaker; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/parser/SqlParserTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/parser/SqlParserTest.java similarity index 96% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/parser/SqlParserTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/parser/SqlParserTest.java index 1a9cd9bd74..65077883fb 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/parser/SqlParserTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/parser/SqlParserTest.java @@ -13,32 +13,32 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.parser; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.parser; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; import com.alibaba.druid.sql.ast.statement.SQLUnionOperator; import com.alibaba.druid.sql.ast.statement.SQLUnionQuery; -import com.amazon.opendistroforelasticsearch.sql.domain.Condition; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.domain.From; -import com.amazon.opendistroforelasticsearch.sql.domain.JoinSelect; -import com.amazon.opendistroforelasticsearch.sql.domain.MethodField; -import com.amazon.opendistroforelasticsearch.sql.domain.Order; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.domain.hints.Hint; -import com.amazon.opendistroforelasticsearch.sql.domain.hints.HintType; -import com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlFeatureNotImplementedException; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.parser.ElasticSqlExprParser; -import com.amazon.opendistroforelasticsearch.sql.parser.ScriptFilter; -import com.amazon.opendistroforelasticsearch.sql.parser.SqlParser; -import com.amazon.opendistroforelasticsearch.sql.query.AggregationQueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.maker.QueryMaker; -import com.amazon.opendistroforelasticsearch.sql.query.multi.MultiQuerySelect; -import com.amazon.opendistroforelasticsearch.sql.util.CheckScriptContents; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Condition; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.From; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.JoinSelect; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.MethodField; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Order; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints.Hint; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints.HintType; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.TestsConstants; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlFeatureNotImplementedException; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.ElasticSqlExprParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.ScriptFilter; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.SqlParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.AggregationQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.maker.QueryMaker; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.multi.MultiQuerySelect; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.CheckScriptContents; import org.elasticsearch.client.Client; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField; @@ -57,11 +57,11 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_ACCOUNT; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_DOG; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_GAME_OF_THRONES; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_ODBC; -import static com.amazon.opendistroforelasticsearch.sql.util.CheckScriptContents.scriptContainsString; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.TestsConstants.TEST_INDEX_ACCOUNT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.TestsConstants.TEST_INDEX_DOG; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.TestsConstants.TEST_INDEX_GAME_OF_THRONES; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.TestsConstants.TEST_INDEX_ODBC; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.CheckScriptContents.scriptContainsString; import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/parser/SubQueryParserTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/parser/SubQueryParserTest.java similarity index 91% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/parser/SubQueryParserTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/parser/SubQueryParserTest.java index cd0c1168c8..87cf801ae3 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/parser/SubQueryParserTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/parser/SubQueryParserTest.java @@ -13,18 +13,18 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.parser; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.parser; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; -import com.amazon.opendistroforelasticsearch.sql.domain.Condition; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.parser.ElasticSqlExprParser; -import com.amazon.opendistroforelasticsearch.sql.parser.SqlParser; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Condition; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.ElasticSqlExprParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.SqlParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; import org.junit.Test; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_ACCOUNT; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.TestsConstants.TEST_INDEX_ACCOUNT; import static org.junit.Assert.assertEquals; public class SubQueryParserTest { diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/BindingTupleQueryPlannerExecuteTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/BindingTupleQueryPlannerExecuteTest.java similarity index 90% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/BindingTupleQueryPlannerExecuteTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/BindingTupleQueryPlannerExecuteTest.java index 478f41d877..4ade3565e4 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/BindingTupleQueryPlannerExecuteTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/BindingTupleQueryPlannerExecuteTest.java @@ -13,13 +13,13 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.planner; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.planner; -import com.amazon.opendistroforelasticsearch.sql.domain.ColumnTypeProvider; -import com.amazon.opendistroforelasticsearch.sql.expression.domain.BindingTuple; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.BindingTupleQueryPlanner; -import com.amazon.opendistroforelasticsearch.sql.util.AggregationUtils; -import com.amazon.opendistroforelasticsearch.sql.util.SqlParserUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.ColumnTypeProvider; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.domain.BindingTuple; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.BindingTupleQueryPlanner; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.AggregationUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.SqlParserUtils; import com.google.common.collect.ImmutableMap; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.search.SearchResponse; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/ESActionFactoryTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/ESActionFactoryTest.java similarity index 89% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/ESActionFactoryTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/ESActionFactoryTest.java index 16a8033b63..fb8e1b7fc0 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/ESActionFactoryTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/ESActionFactoryTest.java @@ -13,11 +13,11 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.planner; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.planner; -import com.amazon.opendistroforelasticsearch.sql.executor.Format; -import com.amazon.opendistroforelasticsearch.sql.query.ESActionFactory; -import com.amazon.opendistroforelasticsearch.sql.util.SqlParserUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.Format; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.ESActionFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.SqlParserUtils; import org.junit.Test; import static org.junit.Assert.assertFalse; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/QueryPlannerBatchTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/QueryPlannerBatchTest.java similarity index 95% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/QueryPlannerBatchTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/QueryPlannerBatchTest.java index 6b78bb62bd..04b21c0f66 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/QueryPlannerBatchTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/QueryPlannerBatchTest.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.planner; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.planner; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -29,9 +29,9 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.runners.Parameterized.Parameters; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.hit; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.hits; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.kv; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.MatcherUtils.hit; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.MatcherUtils.hits; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.MatcherUtils.kv; /** * Batch prefetch testing. Test against different combination of algorithm block size and scroll page size. diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/QueryPlannerConfigTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/QueryPlannerConfigTest.java similarity index 94% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/QueryPlannerConfigTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/QueryPlannerConfigTest.java index 61c42c12d3..41703d7985 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/QueryPlannerConfigTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/QueryPlannerConfigTest.java @@ -13,14 +13,14 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.planner; - -import com.amazon.opendistroforelasticsearch.sql.domain.hints.Hint; -import com.amazon.opendistroforelasticsearch.sql.domain.hints.HintFactory; -import com.amazon.opendistroforelasticsearch.sql.domain.hints.HintType; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.query.planner.HashJoinQueryPlanRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.Config; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.planner; + +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints.Hint; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints.HintFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.hints.HintType; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.HashJoinQueryPlanRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.Config; import org.hamcrest.Matcher; import org.junit.Test; @@ -30,7 +30,7 @@ import static org.hamcrest.Matchers.both; import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.is; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.featureValueOf; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.MatcherUtils.featureValueOf; /** * Hint & Configuring Ability Test Cases diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/QueryPlannerExecuteTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/QueryPlannerExecuteTest.java similarity index 98% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/QueryPlannerExecuteTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/QueryPlannerExecuteTest.java index 7fbfed38fa..b25a87c43f 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/QueryPlannerExecuteTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/QueryPlannerExecuteTest.java @@ -13,15 +13,15 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.planner; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.planner; -import com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.MatcherUtils; import org.elasticsearch.search.SearchHit; import org.hamcrest.MatcherAssert; import org.junit.Test; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.hit; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.hits; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.MatcherUtils.hit; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.MatcherUtils.hits; /** * Query planner execution unit test diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/QueryPlannerExplainTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/QueryPlannerExplainTest.java similarity index 91% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/QueryPlannerExplainTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/QueryPlannerExplainTest.java index 4bd1a048fd..263a92d8f1 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/QueryPlannerExplainTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/QueryPlannerExplainTest.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.planner; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.planner; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.QueryPlanner; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.QueryPlanner; import org.junit.Test; /** diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/QueryPlannerMonitorTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/QueryPlannerMonitorTest.java similarity index 92% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/QueryPlannerMonitorTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/QueryPlannerMonitorTest.java index 5aa4739543..ab51fabf98 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/QueryPlannerMonitorTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/QueryPlannerMonitorTest.java @@ -13,17 +13,17 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.planner; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.planner; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.QueryPlanner; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.QueryPlanner; import org.elasticsearch.search.SearchHits; import org.junit.Ignore; import org.junit.Test; import org.mockito.Spy; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import com.amazon.opendistroforelasticsearch.sql.query.planner.resource.Stats; -import com.amazon.opendistroforelasticsearch.sql.query.planner.resource.Stats.MemStats; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.resource.Stats; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.resource.Stats.MemStats; import static org.mockito.Mockito.doAnswer; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/QueryPlannerTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/QueryPlannerTest.java similarity index 85% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/QueryPlannerTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/QueryPlannerTest.java index d4ff1eee14..c334d43057 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/QueryPlannerTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/QueryPlannerTest.java @@ -13,33 +13,25 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.planner; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.planner; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; import com.alibaba.druid.sql.parser.ParserException; import com.alibaba.druid.sql.parser.SQLExprParser; import com.alibaba.druid.sql.parser.Token; -import com.amazon.opendistroforelasticsearch.sql.domain.JoinSelect; -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.parser.ElasticSqlExprParser; -import com.amazon.opendistroforelasticsearch.sql.parser.SqlParser; -import com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings; -import com.amazon.opendistroforelasticsearch.sql.query.QueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.SqlElasticRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.join.ESJoinQueryActionFactory; -import com.amazon.opendistroforelasticsearch.sql.query.planner.HashJoinQueryPlanRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.QueryPlanner; -import com.amazon.opendistroforelasticsearch.sql.request.SqlRequest; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.core.config.Configurator; -import org.apache.logging.log4j.core.config.builder.api.AppenderComponentBuilder; -import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder; -import org.apache.logging.log4j.core.config.builder.api.LayoutComponentBuilder; -import org.apache.logging.log4j.core.config.builder.api.LoggerComponentBuilder; -import org.apache.logging.log4j.core.config.builder.api.RootLoggerComponentBuilder; -import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.JoinSelect; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.ElasticSqlExprParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.SqlParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.QueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.SqlElasticRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.join.ESJoinQueryActionFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.HashJoinQueryPlanRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.QueryPlanner; +import com.amazon.opendistroforelasticsearch.sql.legacy.request.SqlRequest; import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.TotalHits.Relation; import org.elasticsearch.action.ActionFuture; @@ -53,7 +45,6 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Ignore; import org.mockito.Mock; import org.mockito.MockitoAnnotations; @@ -64,7 +55,6 @@ import java.util.List; import static java.util.Collections.emptyList; -import static org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilderFactory.newConfigurationBuilder; import static org.mockito.Matchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; @@ -89,6 +79,7 @@ public abstract class QueryPlannerTest { private SearchResponse response2; private static final String SCROLL_ID2 = "2"; + /* @BeforeClass public static void initLogger() { ConfigurationBuilder builder = newConfigurationBuilder(); @@ -109,6 +100,7 @@ public static void initLogger() { Configurator.initialize(builder.build()); } + */ @Before public void init() { diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/converter/SQLAggregationParserTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/converter/SQLAggregationParserTest.java similarity index 94% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/converter/SQLAggregationParserTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/converter/SQLAggregationParserTest.java index 2d089012f0..814f24132d 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/converter/SQLAggregationParserTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/converter/SQLAggregationParserTest.java @@ -13,22 +13,19 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.planner.converter; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.planner.converter; import com.alibaba.druid.sql.SQLUtils; -import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLAggregateExpr; -import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; import com.alibaba.druid.sql.ast.statement.SQLSelectItem; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock; -import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitorAdapter; import com.alibaba.druid.util.JdbcConstants; -import com.amazon.opendistroforelasticsearch.sql.domain.ColumnTypeProvider; -import com.amazon.opendistroforelasticsearch.sql.expression.core.Expression; -import com.amazon.opendistroforelasticsearch.sql.expression.core.ExpressionFactory; -import com.amazon.opendistroforelasticsearch.sql.query.planner.converter.SQLAggregationParser; -import com.amazon.opendistroforelasticsearch.sql.query.planner.core.ColumnNode; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.ColumnTypeProvider; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.Expression; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.ExpressionFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.converter.SQLAggregationParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.core.ColumnNode; import org.hamcrest.Description; import org.hamcrest.TypeSafeMatcher; import org.junit.Rule; @@ -37,14 +34,13 @@ import org.junit.runner.RunWith; import org.mockito.runners.MockitoJUnitRunner; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.ExpressionFactory.cast; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.ExpressionFactory.of; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.ADD; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.LOG; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.ExpressionFactory.cast; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.ExpressionFactory.of; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.ADD; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.LOG; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/converter/SQLExprToExpressionConverterTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/converter/SQLExprToExpressionConverterTest.java similarity index 85% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/converter/SQLExprToExpressionConverterTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/converter/SQLExprToExpressionConverterTest.java index 54a8b938af..0c7ea6f9c5 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/converter/SQLExprToExpressionConverterTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/converter/SQLExprToExpressionConverterTest.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.planner.converter; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.planner.converter; import com.alibaba.druid.sql.ast.expr.SQLAggregateExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; @@ -21,25 +21,25 @@ import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; import com.alibaba.druid.sql.ast.expr.SQLIntegerExpr; import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr; -import com.amazon.opendistroforelasticsearch.sql.expression.core.Expression; -import com.amazon.opendistroforelasticsearch.sql.query.planner.converter.SQLAggregationParser; -import com.amazon.opendistroforelasticsearch.sql.query.planner.converter.SQLExprToExpressionConverter; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.Expression; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.converter.SQLAggregationParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.converter.SQLExprToExpressionConverter; import com.google.common.collect.ImmutableMap; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; -import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.junit.MockitoJUnitRunner; import java.util.Arrays; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.ExpressionFactory.literal; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.ExpressionFactory.of; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.ExpressionFactory.ref; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.ADD; -import static com.amazon.opendistroforelasticsearch.sql.expression.core.operator.ScalarOperation.LOG; -import static com.amazon.opendistroforelasticsearch.sql.expression.model.ExprValueFactory.integerValue; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.ExpressionFactory.literal; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.ExpressionFactory.of; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.ExpressionFactory.ref; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.ADD; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.core.operator.ScalarOperation.LOG; +import static com.amazon.opendistroforelasticsearch.sql.legacy.expression.model.ExprValueFactory.integerValue; import static org.junit.Assert.assertEquals; @RunWith(MockitoJUnitRunner.class) diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/converter/SQLToOperatorConverterTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/converter/SQLToOperatorConverterTest.java similarity index 83% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/converter/SQLToOperatorConverterTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/converter/SQLToOperatorConverterTest.java index 6d40daceae..7ab98e9fca 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/converter/SQLToOperatorConverterTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/converter/SQLToOperatorConverterTest.java @@ -13,16 +13,16 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.planner.converter; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.planner.converter; import com.alibaba.druid.sql.SQLUtils; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; import com.alibaba.druid.util.JdbcConstants; -import com.amazon.opendistroforelasticsearch.sql.domain.ColumnTypeProvider; -import com.amazon.opendistroforelasticsearch.sql.expression.domain.BindingTuple; -import com.amazon.opendistroforelasticsearch.sql.query.planner.converter.SQLToOperatorConverter; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.PhysicalOperator; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.project.PhysicalProject; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.ColumnTypeProvider; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.domain.BindingTuple; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.converter.SQLToOperatorConverter; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.PhysicalOperator; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.project.PhysicalProject; import org.elasticsearch.client.Client; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/physical/SearchAggregationResponseHelperTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/physical/SearchAggregationResponseHelperTest.java similarity index 96% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/physical/SearchAggregationResponseHelperTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/physical/SearchAggregationResponseHelperTest.java index 6f9348b1ac..1fc5350a7c 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/physical/SearchAggregationResponseHelperTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/planner/physical/SearchAggregationResponseHelperTest.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.planner.physical; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.planner.physical; -import com.amazon.opendistroforelasticsearch.sql.expression.domain.BindingTuple; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.scroll.BindingTupleRow; -import com.amazon.opendistroforelasticsearch.sql.query.planner.physical.node.scroll.SearchAggregationResponseHelper; -import com.amazon.opendistroforelasticsearch.sql.util.AggregationUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.expression.domain.BindingTuple; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.scroll.BindingTupleRow; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.planner.physical.node.scroll.SearchAggregationResponseHelper; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.AggregationUtils; import com.google.common.collect.ImmutableMap; import org.hamcrest.Matcher; import org.junit.Test; @@ -28,7 +28,7 @@ import java.util.List; import java.util.Map; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.featureValueOf; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.MatcherUtils.featureValueOf; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.contains; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/query/DefaultQueryActionTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/query/DefaultQueryActionTest.java similarity index 88% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/query/DefaultQueryActionTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/query/DefaultQueryActionTest.java index e9873df0d5..d92448e12e 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/query/DefaultQueryActionTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/query/DefaultQueryActionTest.java @@ -13,18 +13,18 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.query; - -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.domain.KVValue; -import com.amazon.opendistroforelasticsearch.sql.domain.MethodField; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.executor.Format; -import com.amazon.opendistroforelasticsearch.sql.metrics.Metrics; -import com.amazon.opendistroforelasticsearch.sql.query.DefaultQueryAction; -import com.amazon.opendistroforelasticsearch.sql.request.SqlRequest; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.query; + +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.KVValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.MethodField; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.Format; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.Metrics; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.DefaultQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.request.SqlRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.client.Client; import org.elasticsearch.common.unit.TimeValue; @@ -40,11 +40,11 @@ import java.util.List; import java.util.Optional; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.CURSOR_ENABLED; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.CURSOR_FETCH_SIZE; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.CURSOR_KEEPALIVE; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.METRICS_ROLLING_WINDOW; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.METRICS_ROLLING_INTERVAL; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.CURSOR_ENABLED; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.CURSOR_FETCH_SIZE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.CURSOR_KEEPALIVE; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.METRICS_ROLLING_WINDOW; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.METRICS_ROLLING_INTERVAL; import static org.hamcrest.Matchers.equalTo; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/RewriteRuleExecutorTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/RewriteRuleExecutorTest.java similarity index 88% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/RewriteRuleExecutorTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/RewriteRuleExecutorTest.java index c8efc83e6c..ff3f3379ef 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/RewriteRuleExecutorTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/RewriteRuleExecutorTest.java @@ -13,11 +13,11 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.rewriter; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.rewriter; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; -import com.amazon.opendistroforelasticsearch.sql.rewriter.RewriteRule; -import com.amazon.opendistroforelasticsearch.sql.rewriter.RewriteRuleExecutor; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.RewriteRule; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.RewriteRuleExecutor; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/identifier/UnquoteIdentifierRuleTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/identifier/UnquoteIdentifierRuleTest.java similarity index 92% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/identifier/UnquoteIdentifierRuleTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/identifier/UnquoteIdentifierRuleTest.java index 5599ce774e..634105ed9b 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/identifier/UnquoteIdentifierRuleTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/identifier/UnquoteIdentifierRuleTest.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.rewriter.identifier; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.rewriter.identifier; import com.alibaba.druid.sql.SQLUtils; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; -import com.amazon.opendistroforelasticsearch.sql.rewriter.identifier.UnquoteIdentifierRule; -import com.amazon.opendistroforelasticsearch.sql.util.SqlParserUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.identifier.UnquoteIdentifierRule; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.SqlParserUtils; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/inline/AliasInliningTests.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/inline/AliasInliningTests.java similarity index 86% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/inline/AliasInliningTests.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/inline/AliasInliningTests.java index 449198e02e..4fea081adc 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/inline/AliasInliningTests.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/inline/AliasInliningTests.java @@ -13,28 +13,25 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.rewriter.inline; - -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; -import com.amazon.opendistroforelasticsearch.sql.esintgtest.SQLIntegTestCase; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.parser.SqlParser; -import com.amazon.opendistroforelasticsearch.sql.query.AggregationQueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.DefaultQueryAction; -import com.amazon.opendistroforelasticsearch.sql.request.SqlRequest; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.rewriter.inline; + +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.SqlParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.AggregationQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.DefaultQueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.request.SqlRequest; import com.google.common.base.Charsets; import com.google.common.io.Resources; import org.elasticsearch.client.Client; import org.json.JSONObject; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import java.io.IOException; import java.net.URL; -import static com.amazon.opendistroforelasticsearch.sql.util.CheckScriptContents.mockLocalClusterState; -import static com.amazon.opendistroforelasticsearch.sql.util.SqlParserUtils.parse; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.CheckScriptContents.mockLocalClusterState; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.SqlParserUtils.parse; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/ordinal/OrdinalRewriterRuleTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/ordinal/OrdinalRewriterRuleTest.java similarity index 93% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/ordinal/OrdinalRewriterRuleTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/ordinal/OrdinalRewriterRuleTest.java index 3656b976a8..4af24ca14a 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/ordinal/OrdinalRewriterRuleTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/ordinal/OrdinalRewriterRuleTest.java @@ -13,14 +13,14 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.rewriter.ordinal; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.rewriter.ordinal; import com.alibaba.druid.sql.SQLUtils; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; -import com.amazon.opendistroforelasticsearch.sql.rewriter.matchtoterm.VerificationException; -import com.amazon.opendistroforelasticsearch.sql.rewriter.ordinal.OrdinalRewriterRule; -import com.amazon.opendistroforelasticsearch.sql.util.SqlParserUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.matchtoterm.VerificationException; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.ordinal.OrdinalRewriterRule; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.SqlParserUtils; import org.junit.Assert; import org.junit.Rule; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/parent/SQLExprParentSetterRuleTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/parent/SQLExprParentSetterRuleTest.java similarity index 86% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/parent/SQLExprParentSetterRuleTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/parent/SQLExprParentSetterRuleTest.java index d3f13738a2..d91f89f689 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/parent/SQLExprParentSetterRuleTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/parent/SQLExprParentSetterRuleTest.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.rewriter.parent; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.rewriter.parent; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; -import com.amazon.opendistroforelasticsearch.sql.rewriter.parent.SQLExprParentSetterRule; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.parent.SQLExprParentSetterRule; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/parent/SQLExprParentSetterTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/parent/SQLExprParentSetterTest.java similarity index 92% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/parent/SQLExprParentSetterTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/parent/SQLExprParentSetterTest.java index f7c8ec1635..c4347d35c6 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/parent/SQLExprParentSetterTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/parent/SQLExprParentSetterTest.java @@ -13,13 +13,13 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.rewriter.parent; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.rewriter.parent; import com.alibaba.druid.sql.ast.expr.SQLInListExpr; import com.alibaba.druid.sql.ast.expr.SQLInSubQueryExpr; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitorAdapter; -import com.amazon.opendistroforelasticsearch.sql.util.SqlParserUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.SqlParserUtils; import org.junit.Test; import static org.junit.Assert.assertNotNull; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/subquery/ExistsSubQueryRewriterTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/subquery/ExistsSubQueryRewriterTest.java similarity index 98% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/subquery/ExistsSubQueryRewriterTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/subquery/ExistsSubQueryRewriterTest.java index cc0234b757..54b39e29e5 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/subquery/ExistsSubQueryRewriterTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/subquery/ExistsSubQueryRewriterTest.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.rewriter.subquery; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.rewriter.subquery; import org.junit.Rule; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/subquery/InSubqueryRewriterTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/subquery/InSubqueryRewriterTest.java similarity index 97% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/subquery/InSubqueryRewriterTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/subquery/InSubqueryRewriterTest.java index 0761cbbc05..11cb83202a 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/subquery/InSubqueryRewriterTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/subquery/InSubqueryRewriterTest.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.rewriter.subquery; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.rewriter.subquery; import org.junit.Rule; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/subquery/NestedQueryContextTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/subquery/NestedQueryContextTest.java similarity index 94% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/subquery/NestedQueryContextTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/subquery/NestedQueryContextTest.java index a1dbd3513c..5d0e72bcb2 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/subquery/NestedQueryContextTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/subquery/NestedQueryContextTest.java @@ -13,13 +13,13 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.rewriter.subquery; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.rewriter.subquery; import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; import com.alibaba.druid.sql.ast.statement.SQLExprTableSource; import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource; import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource.JoinType; -import com.amazon.opendistroforelasticsearch.sql.rewriter.subquery.NestedQueryContext; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery.NestedQueryContext; import org.junit.Test; import static org.junit.Assert.assertFalse; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/subquery/SubQueryRewriteRuleTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/subquery/SubQueryRewriteRuleTest.java similarity index 93% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/subquery/SubQueryRewriteRuleTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/subquery/SubQueryRewriteRuleTest.java index 3e54ab68da..b1ab46ef97 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/subquery/SubQueryRewriteRuleTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/subquery/SubQueryRewriteRuleTest.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.rewriter.subquery; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.rewriter.subquery; -import com.amazon.opendistroforelasticsearch.sql.rewriter.subquery.SubQueryRewriteRule; -import com.amazon.opendistroforelasticsearch.sql.util.SqlParserUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery.SubQueryRewriteRule; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.SqlParserUtils; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/subquery/SubQueryRewriterTestBase.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/subquery/SubQueryRewriterTestBase.java similarity index 82% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/subquery/SubQueryRewriterTestBase.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/subquery/SubQueryRewriterTestBase.java index 1b5cee0095..147c81d523 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/subquery/SubQueryRewriterTestBase.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/subquery/SubQueryRewriterTestBase.java @@ -13,13 +13,13 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.rewriter.subquery; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.rewriter.subquery; import com.alibaba.druid.sql.SQLUtils; import com.alibaba.druid.sql.ast.SQLObject; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; -import com.amazon.opendistroforelasticsearch.sql.rewriter.subquery.SubQueryRewriteRule; -import com.amazon.opendistroforelasticsearch.sql.util.SqlParserUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery.SubQueryRewriteRule; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.SqlParserUtils; public abstract class SubQueryRewriterTestBase { diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/subquery/rewriter/SubqueryAliasRewriterTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/subquery/rewriter/SubqueryAliasRewriterTest.java similarity index 94% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/subquery/rewriter/SubqueryAliasRewriterTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/subquery/rewriter/SubqueryAliasRewriterTest.java index 3bdfcf857b..3bcf2d7fe7 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/subquery/rewriter/SubqueryAliasRewriterTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/subquery/rewriter/SubqueryAliasRewriterTest.java @@ -13,15 +13,15 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.rewriter.subquery.rewriter; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.rewriter.subquery.rewriter; import com.alibaba.druid.sql.SQLUtils; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; -import com.amazon.opendistroforelasticsearch.sql.rewriter.subquery.rewriter.SubqueryAliasRewriter; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery.rewriter.SubqueryAliasRewriter; import org.junit.Test; -import static com.amazon.opendistroforelasticsearch.sql.util.SqlParserUtils.parse; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.SqlParserUtils.parse; import static org.junit.Assert.assertEquals; public class SubqueryAliasRewriterTest { diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/subquery/utils/FindSubQueryTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/subquery/utils/FindSubQueryTest.java similarity index 88% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/subquery/utils/FindSubQueryTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/subquery/utils/FindSubQueryTest.java index 36e6582103..17104ffca6 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/subquery/utils/FindSubQueryTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/subquery/utils/FindSubQueryTest.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.rewriter.subquery.utils; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.rewriter.subquery.utils; -import com.amazon.opendistroforelasticsearch.sql.rewriter.subquery.utils.FindSubQuery; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.subquery.utils.FindSubQuery; import org.junit.Test; -import static com.amazon.opendistroforelasticsearch.sql.util.SqlParserUtils.parse; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.SqlParserUtils.parse; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/term/TermFieldRewriterTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/term/TermFieldRewriterTest.java similarity index 88% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/term/TermFieldRewriterTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/term/TermFieldRewriterTest.java index be62f8a90b..b963131ca2 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/rewriter/term/TermFieldRewriterTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/rewriter/term/TermFieldRewriterTest.java @@ -13,21 +13,21 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.rewriter.term; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.rewriter.term; import com.alibaba.druid.sql.SQLUtils; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; -import com.amazon.opendistroforelasticsearch.sql.rewriter.matchtoterm.TermFieldRewriter; -import com.amazon.opendistroforelasticsearch.sql.rewriter.matchtoterm.VerificationException; -import com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils; -import com.amazon.opendistroforelasticsearch.sql.util.SqlParserUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.matchtoterm.TermFieldRewriter; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.matchtoterm.VerificationException; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.MatcherUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.SqlParserUtils; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import static com.amazon.opendistroforelasticsearch.sql.util.MultipleIndexClusterUtils.mockMultipleIndexEnv; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.MultipleIndexClusterUtils.mockMultipleIndexEnv; import static org.hamcrest.MatcherAssert.assertThat; public class TermFieldRewriterTest { diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/spatial/WktToGeoJsonConverterTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/spatial/WktToGeoJsonConverterTest.java similarity index 98% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/spatial/WktToGeoJsonConverterTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/spatial/WktToGeoJsonConverterTest.java index 3958908b21..27976b884e 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/spatial/WktToGeoJsonConverterTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/spatial/WktToGeoJsonConverterTest.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.spatial; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.spatial; -import com.amazon.opendistroforelasticsearch.sql.spatial.WktToGeoJsonConverter; +import com.amazon.opendistroforelasticsearch.sql.legacy.spatial.WktToGeoJsonConverter; import org.junit.Test; import org.junit.Assert; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/BackticksUnquoterTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/BackticksUnquoterTest.java similarity index 83% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/BackticksUnquoterTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/BackticksUnquoterTest.java index 2e9bfda1a2..2330742728 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/BackticksUnquoterTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/BackticksUnquoterTest.java @@ -13,13 +13,13 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.utils; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.utils; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; import org.junit.Test; -import static com.amazon.opendistroforelasticsearch.sql.utils.StringUtils.unquoteFullColumn; -import static com.amazon.opendistroforelasticsearch.sql.utils.StringUtils.unquoteSingleField; +import static com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils.unquoteFullColumn; +import static com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils.unquoteSingleField; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.IsEqual.equalTo; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/LogUtilsTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/LogUtilsTest.java similarity index 94% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/LogUtilsTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/LogUtilsTest.java index 4b0b51cf3e..7cd5e354ae 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/LogUtilsTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/LogUtilsTest.java @@ -13,11 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.utils; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.utils; -import com.amazon.opendistroforelasticsearch.sql.utils.LogUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.LogUtils; import org.apache.logging.log4j.ThreadContext; -import org.hamcrest.Matchers; import org.junit.After; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/PrettyFormatterTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/PrettyFormatterTest.java similarity index 91% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/PrettyFormatterTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/PrettyFormatterTest.java index 915b40fc9d..653f1229f8 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/PrettyFormatterTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/PrettyFormatterTest.java @@ -13,10 +13,10 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.utils; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.utils; -import com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils; -import com.amazon.opendistroforelasticsearch.sql.utils.JsonPrettyFormatter; +import com.amazon.opendistroforelasticsearch.sql.legacy.util.TestUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.JsonPrettyFormatter; import com.google.common.io.Files; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/QueryDataAnonymizerTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/QueryDataAnonymizerTest.java similarity index 96% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/QueryDataAnonymizerTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/QueryDataAnonymizerTest.java index 49c2134497..e4cdbaf2c8 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/QueryDataAnonymizerTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/QueryDataAnonymizerTest.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.utils; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.utils; -import com.amazon.opendistroforelasticsearch.sql.utils.QueryDataAnonymizer; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.QueryDataAnonymizer; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/SQLFunctionsTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/SQLFunctionsTest.java similarity index 81% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/SQLFunctionsTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/SQLFunctionsTest.java index 6d48f5c004..f324e3f9e1 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/SQLFunctionsTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/SQLFunctionsTest.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.utils; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.utils; import com.alibaba.druid.sql.ast.SQLDataType; import com.alibaba.druid.sql.ast.SQLDataTypeImpl; @@ -21,20 +21,17 @@ import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; import com.alibaba.druid.sql.ast.expr.SQLIntegerExpr; import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr; -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.base.ESDataType; -import com.amazon.opendistroforelasticsearch.sql.domain.ColumnTypeProvider; -import com.amazon.opendistroforelasticsearch.sql.domain.KVValue; -import com.amazon.opendistroforelasticsearch.sql.domain.MethodField; -import com.amazon.opendistroforelasticsearch.sql.domain.ScriptMethodField; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.executor.format.Schema; -import com.amazon.opendistroforelasticsearch.sql.parser.FieldMaker; -import com.amazon.opendistroforelasticsearch.sql.utils.SQLFunctions; +import com.amazon.opendistroforelasticsearch.sql.legacy.antlr.semantic.types.base.ESDataType; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.ColumnTypeProvider; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.KVValue; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.MethodField; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.ScriptMethodField; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.format.Schema; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.SQLFunctions; import com.google.common.collect.ImmutableList; import org.elasticsearch.common.collect.Tuple; import org.junit.Assert; -import org.junit.Before; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/StringUtilsTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/StringUtilsTest.java similarity index 94% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/StringUtilsTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/StringUtilsTest.java index e91c018d70..ea4417f843 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/StringUtilsTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/StringUtilsTest.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.utils; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.utils; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/UtilTest.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/UtilTest.java similarity index 94% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/UtilTest.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/UtilTest.java index c2f8e024d5..a063b7bd75 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/UtilTest.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/unittest/utils/UtilTest.java @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.unittest.utils; +package com.amazon.opendistroforelasticsearch.sql.legacy.unittest.utils; -import com.amazon.opendistroforelasticsearch.sql.utils.Util; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.Util; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/AggregationUtils.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/AggregationUtils.java similarity index 98% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/util/AggregationUtils.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/AggregationUtils.java index 165a8ed31e..0f270fb007 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/AggregationUtils.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/AggregationUtils.java @@ -13,10 +13,9 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.util; +package com.amazon.opendistroforelasticsearch.sql.legacy.util; import com.google.common.collect.ImmutableMap; -import lombok.SneakyThrows; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.xcontent.ContextParser; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/CheckScriptContents.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/CheckScriptContents.java similarity index 88% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/util/CheckScriptContents.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/CheckScriptContents.java index 72e6a5809e..900e40c499 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/CheckScriptContents.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/CheckScriptContents.java @@ -13,24 +13,23 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.util; +package com.amazon.opendistroforelasticsearch.sql.legacy.util; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; import com.alibaba.druid.sql.parser.ParserException; -import com.amazon.opendistroforelasticsearch.sql.domain.Condition; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; -import com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.parser.ElasticSqlExprParser; -import com.amazon.opendistroforelasticsearch.sql.parser.ScriptFilter; -import com.amazon.opendistroforelasticsearch.sql.parser.SqlParser; -import com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings; -import com.amazon.opendistroforelasticsearch.sql.query.ESActionFactory; -import com.amazon.opendistroforelasticsearch.sql.query.QueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.SqlElasticRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Condition; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.legacy.domain.Where; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.ElasticSqlExprParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.ScriptFilter; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.SqlParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.ESActionFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.QueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.SqlElasticRequestBuilder; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse; @@ -39,10 +38,10 @@ import org.elasticsearch.client.Client; import org.elasticsearch.client.IndicesAdminClient; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.xcontent.DeprecationHandler; @@ -239,13 +238,13 @@ public static void mockLocalClusterState(String mappings) { public static ClusterService mockClusterService(String mappings) { ClusterService mockService = mock(ClusterService.class); ClusterState mockState = mock(ClusterState.class); - MetaData mockMetaData = mock(MetaData.class); + Metadata mockMetaData = mock(Metadata.class); when(mockService.state()).thenReturn(mockState); - when(mockState.metaData()).thenReturn(mockMetaData); + when(mockState.metadata()).thenReturn(mockMetaData); try { - ImmutableOpenMap.Builder> builder = ImmutableOpenMap.builder(); - builder.put(TestsConstants.TEST_INDEX_BANK, IndexMetaData.fromXContent(createParser(mappings)).getMappings()); + ImmutableOpenMap.Builder> builder = ImmutableOpenMap.builder(); + builder.put(TestsConstants.TEST_INDEX_BANK, IndexMetadata.fromXContent(createParser(mappings)).getMappings()); when(mockMetaData.findMappings(any(), any(), any())).thenReturn(builder.build()); } catch (IOException e) { diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/HasFieldWithValue.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/HasFieldWithValue.java similarity index 97% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/util/HasFieldWithValue.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/HasFieldWithValue.java index 260dd27308..2d9f415a11 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/HasFieldWithValue.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/HasFieldWithValue.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.util; +package com.amazon.opendistroforelasticsearch.sql.legacy.util; import org.hamcrest.FeatureMatcher; import org.hamcrest.Matcher; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/MatcherUtils.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/MatcherUtils.java similarity index 84% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/util/MatcherUtils.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/MatcherUtils.java index cdf81928cb..66175e9e3c 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/MatcherUtils.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/MatcherUtils.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.util; +package com.amazon.opendistroforelasticsearch.sql.legacy.util; import com.google.common.base.Strings; import org.elasticsearch.search.SearchHit; @@ -36,6 +36,7 @@ import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsInRelativeOrder; import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasEntry; @@ -131,16 +132,27 @@ public static Matcher kvInt(String key, Matcher matcher) { return featureValueOf("Json Match", matcher, actual -> (Integer) actual.query(key)); } - @SuppressWarnings("unchecked") + @SafeVarargs public static void verifySchema(JSONObject response, Matcher... matchers) { verify(response.getJSONArray("schema"), matchers); } - @SuppressWarnings("unchecked") + @SafeVarargs public static void verifyDataRows(JSONObject response, Matcher... matchers) { verify(response.getJSONArray("datarows"), matchers); } + @SafeVarargs + public static void verifyColumn(JSONObject response, Matcher... matchers) { + verify(response.getJSONArray("schema"), matchers); + } + + @SafeVarargs + public static void verifyOrder(JSONObject response, Matcher... matchers) { + verifyOrder(response.getJSONArray("datarows"), matchers); + } + + @SafeVarargs @SuppressWarnings("unchecked") public static void verifyDataRowsInOrder(JSONObject response, Matcher... matchers) { verifyInOrder(response.getJSONArray("datarows"), matchers); @@ -154,6 +166,8 @@ public static void verify(JSONArray array, Matcher... matchers) { assertThat(objects, containsInAnyOrder(matchers)); } + @SafeVarargs + @SuppressWarnings("unchecked") public static void verifyInOrder(JSONArray array, Matcher... matchers) { List objects = new ArrayList<>(); array.iterator().forEachRemaining(o -> objects.add((T) o)); @@ -172,6 +186,14 @@ public static void verifySome(JSONArray array, Matcher... matchers) { } } + @SafeVarargs + public static void verifyOrder(JSONArray array, Matcher... matchers) { + List objects = new ArrayList<>(); + array.iterator().forEachRemaining(o -> objects.add((T) o)); + assertEquals(matchers.length, objects.size()); + assertThat(objects, containsInRelativeOrder(matchers)); + } + public static TypeSafeMatcher schema(String expectedName, String expectedAlias, String expectedType) { return new TypeSafeMatcher() { @Override @@ -209,6 +231,34 @@ protected boolean matchesSafely(JSONArray array) { }; } + public static TypeSafeMatcher columnPattern(String regex) { + return new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(JSONObject jsonObject) { + return ((String)jsonObject.query("/name")).matches(regex); + } + + @Override + public void describeTo(Description description) { + description.appendText(String.format("(column_pattern=%s)", regex)); + } + }; + } + + public static TypeSafeMatcher columnName(String name) { + return new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(JSONObject jsonObject) { + return jsonObject.query("/name").equals(name); + } + + @Override + public void describeTo(Description description) { + description.appendText(String.format("(name=%s)", name)); + } + }; + } + /** * Tests if a string is equal to another string, ignore the case and whitespace. diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/MultipleIndexClusterUtils.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/MultipleIndexClusterUtils.java similarity index 85% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/util/MultipleIndexClusterUtils.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/MultipleIndexClusterUtils.java index 0bd2e3a91c..cbaa0da45c 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/MultipleIndexClusterUtils.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/MultipleIndexClusterUtils.java @@ -13,23 +13,23 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.util; +package com.amazon.opendistroforelasticsearch.sql.legacy.util; -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; import com.google.common.collect.ImmutableMap; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.collect.ImmutableOpenMap; import java.io.IOException; import java.util.Map; -import static com.amazon.opendistroforelasticsearch.sql.util.CheckScriptContents.createParser; -import static com.amazon.opendistroforelasticsearch.sql.util.CheckScriptContents.mockIndexNameExpressionResolver; -import static com.amazon.opendistroforelasticsearch.sql.util.CheckScriptContents.mockSqlSettings; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.CheckScriptContents.createParser; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.CheckScriptContents.mockIndexNameExpressionResolver; +import static com.amazon.opendistroforelasticsearch.sql.legacy.util.CheckScriptContents.mockSqlSettings; import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; @@ -149,7 +149,7 @@ public class MultipleIndexClusterUtils { "}"; public static void mockMultipleIndexEnv() { - mockLocalClusterState(new ImmutableMap.Builder>>() + mockLocalClusterState(new ImmutableMap.Builder>>() .put(INDEX_ACCOUNT_1, buildIndexMapping(INDEX_ACCOUNT_1, INDEX_ACCOUNT_1_MAPPING)) .put(INDEX_ACCOUNT_2, buildIndexMapping(INDEX_ACCOUNT_2, INDEX_ACCOUNT_2_MAPPING)) .put(INDEX_ACCOUNT_ALL, buildIndexMapping(new ImmutableMap.Builder() @@ -159,22 +159,22 @@ public static void mockMultipleIndexEnv() { .build()); } - public static void mockLocalClusterState(Map>> indexMapping) { + public static void mockLocalClusterState(Map>> indexMapping) { LocalClusterState.state().setClusterService(mockClusterService(indexMapping)); LocalClusterState.state().setResolver(mockIndexNameExpressionResolver()); LocalClusterState.state().setSqlSettings(mockSqlSettings()); } - public static ClusterService mockClusterService(Map>> indexMapping) { + public static ClusterService mockClusterService(Map>> indexMapping) { ClusterService mockService = mock(ClusterService.class); ClusterState mockState = mock(ClusterState.class); - MetaData mockMetaData = mock(MetaData.class); + Metadata mockMetaData = mock(Metadata.class); when(mockService.state()).thenReturn(mockState); - when(mockState.metaData()).thenReturn(mockMetaData); + when(mockState.metadata()).thenReturn(mockMetaData); try { - for (Map.Entry>> entry : indexMapping.entrySet()) { + for (Map.Entry>> entry : indexMapping.entrySet()) { when(mockMetaData.findMappings(eq(new String[]{entry.getKey()}), any(), any())).thenReturn(entry.getValue()); } } catch (IOException e) { @@ -183,11 +183,11 @@ public static ClusterService mockClusterService(Map> buildIndexMapping(Map indexMapping) { + private static ImmutableOpenMap> buildIndexMapping(Map indexMapping) { try { - ImmutableOpenMap.Builder> builder = ImmutableOpenMap.builder(); + ImmutableOpenMap.Builder> builder = ImmutableOpenMap.builder(); for (Map.Entry entry : indexMapping.entrySet()) { - builder.put(entry.getKey(), IndexMetaData.fromXContent(createParser(entry.getValue())).getMappings()); + builder.put(entry.getKey(), IndexMetadata.fromXContent(createParser(entry.getValue())).getMappings()); } return builder.build(); } catch (IOException e) { @@ -195,11 +195,11 @@ private static ImmutableOpenMap> buildIndexMapping(String index, + private static ImmutableOpenMap> buildIndexMapping(String index, String mapping) { try { - ImmutableOpenMap.Builder> builder = ImmutableOpenMap.builder(); - builder.put(index, IndexMetaData.fromXContent(createParser(mapping)).getMappings()); + ImmutableOpenMap.Builder> builder = ImmutableOpenMap.builder(); + builder.put(index, IndexMetadata.fromXContent(createParser(mapping)).getMappings()); return builder.build(); } catch (IOException e) { throw new IllegalStateException(e); diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/SqlExplainUtils.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/SqlExplainUtils.java similarity index 82% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/util/SqlExplainUtils.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/SqlExplainUtils.java index f44c01ef20..ac90b2dbda 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/SqlExplainUtils.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/SqlExplainUtils.java @@ -13,12 +13,12 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.util; +package com.amazon.opendistroforelasticsearch.sql.legacy.util; import com.alibaba.druid.sql.parser.ParserException; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.query.ESActionFactory; -import com.amazon.opendistroforelasticsearch.sql.query.QueryAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.ESActionFactory; +import com.amazon.opendistroforelasticsearch.sql.legacy.query.QueryAction; import org.elasticsearch.client.Client; import org.mockito.Mockito; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/SqlParserUtils.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/SqlParserUtils.java similarity index 85% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/util/SqlParserUtils.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/SqlParserUtils.java index 5205d57128..2b4ce9a555 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/SqlParserUtils.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/SqlParserUtils.java @@ -13,14 +13,14 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.util; +package com.amazon.opendistroforelasticsearch.sql.legacy.util; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLQueryExpr; import com.alibaba.druid.sql.parser.ParserException; import com.alibaba.druid.sql.parser.Token; -import com.amazon.opendistroforelasticsearch.sql.parser.ElasticSqlExprParser; -import com.amazon.opendistroforelasticsearch.sql.rewriter.parent.SQLExprParentSetter; +import com.amazon.opendistroforelasticsearch.sql.legacy.parser.ElasticSqlExprParser; +import com.amazon.opendistroforelasticsearch.sql.legacy.rewriter.parent.SQLExprParentSetter; /** * Test utils class include all SQLExpr related method. diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestUtils.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/TestUtils.java similarity index 99% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestUtils.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/TestUtils.java index f0c490dc39..890170cfd3 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestUtils.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/TestUtils.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.esintgtest; +package com.amazon.opendistroforelasticsearch.sql.legacy.util; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestsConstants.java b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/TestsConstants.java similarity index 94% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestsConstants.java rename to legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/TestsConstants.java index 66c01ad244..74075500ce 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestsConstants.java +++ b/legacy/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/util/TestsConstants.java @@ -13,13 +13,16 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.sql.esintgtest; +package com.amazon.opendistroforelasticsearch.sql.legacy.util; /** * Created by omershelef on 18/12/14. */ public class TestsConstants { + public final static String PERSISTENT = "persistent"; + public final static String TRANSIENT = "transient"; + public final static String TEST_INDEX = "elasticsearch-sql_test_index"; public final static String TEST_INDEX_ONLINE = TEST_INDEX + "_online"; diff --git a/legacy/src/test/resources/expectedOutput/aggregation_query_explain.json b/legacy/src/test/resources/expectedOutput/aggregation_query_explain.json new file mode 100644 index 0000000000..9675b2b5be --- /dev/null +++ b/legacy/src/test/resources/expectedOutput/aggregation_query_explain.json @@ -0,0 +1,74 @@ +{ + "from" : 0, + "size" : 0, + "_source" : { + "includes" : [ + "address", + "script", + "COUNT" + ], + "excludes" : [ ] + }, + "stored_fields" : [ + "address", + "a2345" + ], + "script_fields" : { + "a2345" : { + "script" : { + "source" : "if((doc['gender'].value=='0')){'aaa'} else {'bbb'}", + "lang" : "painless" + }, + "ignore_failure" : false + } + }, + "aggregations" : { + "terms(field=address,execution_hint=global_ordinals)" : { + "terms" : { + "field" : "address", + "size" : 10, + "min_doc_count" : 1, + "shard_min_doc_count" : 0, + "show_term_doc_count_error" : false, + "execution_hint": "global_ordinals", + "order" : [ + { + "_count" : "desc" + }, + { + "_key" : "asc" + } + ] + }, + "aggregations" : { + "a2345" : { + "terms" : { + "script" : { + "source" : "if((doc['gender'].value=='0')){'aaa'} else {'bbb'}", + "lang" : "painless" + }, + "size" : 10, + "min_doc_count" : 1, + "shard_min_doc_count" : 0, + "show_term_doc_count_error" : false, + "order" : [ + { + "_count" : "desc" + }, + { + "_key" : "asc" + } + ] + }, + "aggregations" : { + "COUNT(age)" : { + "value_count" : { + "field" : "age" + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/legacy/src/test/resources/expectedOutput/delete_explain.json b/legacy/src/test/resources/expectedOutput/delete_explain.json new file mode 100644 index 0000000000..a9e173f2b3 --- /dev/null +++ b/legacy/src/test/resources/expectedOutput/delete_explain.json @@ -0,0 +1,39 @@ +{ + "size" : 1000, + "query" : { + "bool" : { + "must" : [ + { + "bool" : { + "must" : [ + { + "wildcard" : { + "firstname" : { + "wildcard" : "A*", + "boost" : 1.0 + } + } + }, + { + "range" : { + "age" : { + "from" : 20, + "to" : null, + "include_lower" : false, + "include_upper" : true, + "boost" : 1.0 + } + } + } + ], + "adjust_pure_negative" : true, + "boost" : 1.0 + } + } + ], + "adjust_pure_negative" : true, + "boost" : 1.0 + } + }, + "_source" : false +} \ No newline at end of file diff --git a/src/test/resources/expectedOutput/explain_format_pretty.json b/legacy/src/test/resources/expectedOutput/explain_format_pretty.json similarity index 100% rename from src/test/resources/expectedOutput/explain_format_pretty.json rename to legacy/src/test/resources/expectedOutput/explain_format_pretty.json diff --git a/src/test/resources/expectedOutput/json_filter_explain.json b/legacy/src/test/resources/expectedOutput/json_filter_explain.json similarity index 100% rename from src/test/resources/expectedOutput/json_filter_explain.json rename to legacy/src/test/resources/expectedOutput/json_filter_explain.json diff --git a/legacy/src/test/resources/expectedOutput/search_explain.json b/legacy/src/test/resources/expectedOutput/search_explain.json new file mode 100644 index 0000000000..88b3d7b291 --- /dev/null +++ b/legacy/src/test/resources/expectedOutput/search_explain.json @@ -0,0 +1,65 @@ +{ + "from" : 0, + "size" : 0, + "query" : { + "bool" : { + "must" : [ + { + "bool" : { + "must" : [ + { + "wildcard" : { + "firstname" : { + "wildcard" : "A*", + "boost" : 1.0 + } + } + }, + { + "range" : { + "age" : { + "from" : 20, + "to" : null, + "include_lower" : false, + "include_upper" : true, + "boost" : 1.0 + } + } + } + ], + "adjust_pure_negative" : true, + "boost" : 1.0 + } + } + ], + "adjust_pure_negative" : true, + "boost" : 1.0 + } + }, + "sort" : [ + { + "_score" : { + "order" : "asc" + } + } + ], + "aggregations" : { + "gender" : { + "terms" : { + "field" : "gender", + "size" : 200, + "min_doc_count" : 1, + "shard_min_doc_count" : 0, + "show_term_doc_count_error" : false, + "order" : [ + { + "_count" : "desc" + }, + { + "_key" : "asc" + } + ] + } + } + } +} \ No newline at end of file diff --git a/src/test/resources/expectedOutput/select_where_true.json b/legacy/src/test/resources/expectedOutput/select_where_true.json similarity index 100% rename from src/test/resources/expectedOutput/select_where_true.json rename to legacy/src/test/resources/expectedOutput/select_where_true.json diff --git a/src/test/resources/explain_format_oneline.json b/legacy/src/test/resources/explain_format_oneline.json similarity index 100% rename from src/test/resources/explain_format_oneline.json rename to legacy/src/test/resources/explain_format_oneline.json diff --git a/src/test/resources/mappings/field_mappings.json b/legacy/src/test/resources/mappings/field_mappings.json similarity index 100% rename from src/test/resources/mappings/field_mappings.json rename to legacy/src/test/resources/mappings/field_mappings.json diff --git a/src/test/resources/mappings/semantics.json b/legacy/src/test/resources/mappings/semantics.json similarity index 100% rename from src/test/resources/mappings/semantics.json rename to legacy/src/test/resources/mappings/semantics.json diff --git a/plugin/build.gradle b/plugin/build.gradle new file mode 100644 index 0000000000..71430b95de --- /dev/null +++ b/plugin/build.gradle @@ -0,0 +1,130 @@ +plugins { + id 'java' + id 'jacoco' + id 'elasticsearch.esplugin' +} + +ext { + projectSubstitutions = [:] + licenseFile = rootProject.file('LICENSE.TXT') + noticeFile = rootProject.file('NOTICE') +} + +repositories { + mavenCentral() +} + +esplugin { + name 'opendistro_sql' + description 'Open Distro for Elasticsearch SQL' + classname 'com.amazon.opendistroforelasticsearch.sql.plugin.SQLPlugin' + licenseFile rootProject.file("LICENSE.txt") + noticeFile rootProject.file("NOTICE") +} + +javadoc.enabled = false +test.enabled = false +integTest.enabled = false +dependencyLicenses.enabled = false +thirdPartyAudit.enabled = false + +configurations.all { + // conflict with spring-jcl + exclude group: "commons-logging", module: "commons-logging" +} + +dependencies { + compile group: 'org.springframework', name: 'spring-beans', version: '5.2.5.RELEASE' + compile project(":ppl") + compile project(':legacy') + compile project(':elasticsearch') +} + + +ext { + projectSubstitutions = [:] + licenseFile = rootProject.file('LICENSE.TXT') + noticeFile = rootProject.file('NOTICE') +} + +// ANTLR generated parser file is too large to be checked which caused licenseHeaders stuck. +licenseHeaders { + enabled = true + excludes = ['com/amazon/opendistroforelasticsearch/sql/antlr/parser/**'] +} + +tasks.withType(licenseHeaders.class) { + additionalLicense 'AL ', 'Apache', 'Licensed under the Apache License, Version 2.0 (the "License")' +} + +// TODO: fix forbidden APIs +// from police-man plugin, see https://github.com/policeman-tools/forbidden-apis/wiki/GradleUsage +forbiddenApis.ignoreFailures = true +// TODO: fix forbidden code patterns +// introduced by elasticsearch plugin +// see https://github.com/elastic/elasticsearch/blob/master/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ForbiddenPatternsTask.java +forbiddenPatterns { + setEnabled(false) +} + +// TODO: fix license. skip dependency license checks +dependencyLicenses.enabled = false + +// We don't need to follow ES testing naming conventions. +// see https://github.com/elastic/elasticsearch/blob/323f312bbc829a63056a79ebe45adced5099f6e6/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java +// enable testingConventions check will cause errors like: "Classes ending with [Tests] must subclass [LuceneTestCase]" +testingConventions.enabled = false + +// TODO: need to verify the thirdPartyAudi +// currently it complains missing classes like ibatis, mysql etc, should not be a problem +thirdPartyAudit.enabled = false + + +apply plugin: 'nebula.ospackage' +validateNebulaPom.enabled = false + +// This is afterEvaluate because the bundlePlugin ZIP task is updated afterEvaluate and changes the ZIP name to match the plugin name +afterEvaluate { + ospackage { + packageName = "${name}" + release = isSnapshot ? "0.1" : '1' + version = "${project.version}" - "-SNAPSHOT" + + into '/usr/share/elasticsearch/plugins' + from(zipTree(bundlePlugin.archivePath)) { + into esplugin.name + } + + user 'root' + permissionGroup 'root' + fileMode 0644 + dirMode 0755 + + requires('elasticsearch-oss', versions.elasticsearch, EQUAL) + packager = 'Amazon' + vendor = 'Amazon' + os = 'LINUX' + prefix '/usr' + + license 'ASL-2.0' + maintainer 'OpenDistro for Elasticsearch Team ' + url 'https://opendistro.github.io/elasticsearch/downloads' + summary ''' + SQL plugin for OpenDistro for Elasticsearch. + Reference documentation can be found at https://opendistro.github.io/for-elasticsearch-docs/. + '''.stripIndent().replace('\n', ' ').trim() + } + + buildRpm { + arch = 'NOARCH' + archiveName "${packageName}-${version}.rpm" + dependsOn 'assemble' + } + + buildDeb { + arch = 'amd64' + archiveName "${packageName}-${version}.deb" + dependsOn 'assemble' + } + +} diff --git a/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SQLPlugin.java new file mode 100644 index 0000000000..4fe9b71980 --- /dev/null +++ b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SQLPlugin.java @@ -0,0 +1,147 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.plugin; + +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.setting.ElasticsearchSettings; +import com.amazon.opendistroforelasticsearch.sql.legacy.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.legacy.executor.AsyncRestExecutor; +import com.amazon.opendistroforelasticsearch.sql.legacy.metrics.Metrics; +import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlSettingsAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.RestSqlStatsAction; +import com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings; +import com.amazon.opendistroforelasticsearch.sql.plugin.rest.RestPPLQueryAction; +import com.google.common.collect.ImmutableList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.function.Supplier; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.threadpool.ExecutorBuilder; +import org.elasticsearch.threadpool.FixedExecutorBuilder; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; + +public class SQLPlugin extends Plugin implements ActionPlugin { + + /** + * Sql plugin specific settings in ES cluster settings. + */ + private final SqlSettings sqlSettings = new SqlSettings(); + + private ClusterService clusterService; + + /** + * Settings should be inited when bootstrap the plugin. + */ + private com.amazon.opendistroforelasticsearch.sql.common.setting.Settings pluginSettings; + + public String name() { + return "sql"; + } + + public String description() { + return "Use sql to query elasticsearch."; + } + + @Override + public List getRestHandlers(Settings settings, RestController restController, + ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, + SettingsFilter settingsFilter, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster) { + Objects.requireNonNull(clusterService, "Cluster service is required"); + Objects.requireNonNull(pluginSettings, "Cluster settings is required"); + + LocalClusterState.state().setResolver(indexNameExpressionResolver); + Metrics.getInstance().registerDefaultMetrics(); + + return Arrays.asList( + new RestPPLQueryAction(restController, clusterService, pluginSettings), + new RestSqlAction(settings, clusterService), + new RestSqlStatsAction(settings, restController), + new RestSqlSettingsAction(settings, restController) + ); + } + + @Override + public Collection createComponents(Client client, ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry contentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameResolver, + Supplier + repositoriesServiceSupplier) { + this.clusterService = clusterService; + this.pluginSettings = new ElasticsearchSettings(clusterService.getClusterSettings()); + + LocalClusterState.state().setClusterService(clusterService); + LocalClusterState.state().setSqlSettings(sqlSettings); + + return super + .createComponents(client, clusterService, threadPool, resourceWatcherService, scriptService, + contentRegistry, environment, nodeEnvironment, namedWriteableRegistry, + indexNameResolver, repositoriesServiceSupplier); + } + + @Override + public List> getExecutorBuilders(Settings settings) { + return Collections.singletonList( + new FixedExecutorBuilder( + settings, + AsyncRestExecutor.SQL_WORKER_THREAD_POOL_NAME, + EsExecutors.allocatedProcessors(settings), + 1000, + null + ) + ); + } + + @Override + public List> getSettings() { + ImmutableList> settings = + new ImmutableList.Builder>().addAll(sqlSettings.getSettings()) + .addAll(ElasticsearchSettings.pluginSettings()).build(); + return settings; + } + +} diff --git a/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/request/PPLQueryRequestFactory.java b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/request/PPLQueryRequestFactory.java new file mode 100644 index 0000000000..99eb56e457 --- /dev/null +++ b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/request/PPLQueryRequestFactory.java @@ -0,0 +1,67 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.plugin.request; + +import com.amazon.opendistroforelasticsearch.sql.ppl.domain.PPLQueryRequest; +import org.elasticsearch.rest.RestRequest; +import org.json.JSONException; +import org.json.JSONObject; + +/** + * Factory of {@link PPLQueryRequest}. + */ +public class PPLQueryRequestFactory { + private static final String PPL_URL_PARAM_KEY = "ppl"; + private static final String PPL_FIELD_NAME = "query"; + + /** + * Build {@link PPLQueryRequest} from {@link RestRequest}. + * @param request {@link PPLQueryRequest} + * @return {@link RestRequest} + */ + public static PPLQueryRequest getPPLRequest(RestRequest request) { + switch (request.method()) { + case GET: + return parsePPLRequestFromUrl(request); + case POST: + return parsePPLRequestFromPayload(request); + default: + throw new IllegalArgumentException( + "ES PPL doesn't supported HTTP " + request.method().name()); + } + } + + private static PPLQueryRequest parsePPLRequestFromUrl(RestRequest restRequest) { + String ppl; + + ppl = restRequest.param(PPL_URL_PARAM_KEY); + if (ppl == null) { + throw new IllegalArgumentException("Cannot find ppl parameter from the URL"); + } + return new PPLQueryRequest(ppl, null); + } + + private static PPLQueryRequest parsePPLRequestFromPayload(RestRequest restRequest) { + String content = restRequest.content().utf8ToString(); + JSONObject jsonContent; + try { + jsonContent = new JSONObject(content); + } catch (JSONException e) { + throw new IllegalArgumentException("Failed to parse request payload", e); + } + return new PPLQueryRequest(jsonContent.getString(PPL_FIELD_NAME), jsonContent); + } +} diff --git a/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/ElasticsearchPluginConfig.java b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/ElasticsearchPluginConfig.java new file mode 100644 index 0000000000..cc9de58760 --- /dev/null +++ b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/ElasticsearchPluginConfig.java @@ -0,0 +1,77 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.plugin.rest; + +import com.amazon.opendistroforelasticsearch.sql.common.setting.Settings; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchNodeClient; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor.ElasticsearchExecutionEngine; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor.protector.ElasticsearchExecutionProtector; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor.protector.ExecutionProtector; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.monitor.ElasticsearchMemoryHealthy; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.monitor.ElasticsearchResourceMonitor; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.ElasticsearchStorageEngine; +import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine; +import com.amazon.opendistroforelasticsearch.sql.monitor.ResourceMonitor; +import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.cluster.service.ClusterService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Elasticsearch plugin config that injects cluster service and node client from plugin + * and initialize Elasticsearch storage and execution engine. + */ +@Configuration +public class ElasticsearchPluginConfig { + + @Autowired + private ClusterService clusterService; + + @Autowired + private NodeClient nodeClient; + + @Autowired + private Settings settings; + + @Bean + public ElasticsearchClient client() { + return new ElasticsearchNodeClient(clusterService, nodeClient); + } + + @Bean + public StorageEngine storageEngine() { + return new ElasticsearchStorageEngine(client()); + } + + @Bean + public ExecutionEngine executionEngine() { + return new ElasticsearchExecutionEngine(client(), protector()); + } + + @Bean + public ResourceMonitor resourceMonitor() { + return new ElasticsearchResourceMonitor(settings, new ElasticsearchMemoryHealthy()); + } + + @Bean + public ExecutionProtector protector() { + return new ElasticsearchExecutionProtector(resourceMonitor()); + } +} diff --git a/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLQueryAction.java b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLQueryAction.java new file mode 100644 index 0000000000..7062b31ccb --- /dev/null +++ b/plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/rest/RestPPLQueryAction.java @@ -0,0 +1,146 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.plugin.rest; + +import static com.amazon.opendistroforelasticsearch.sql.protocol.response.format.JsonResponseFormatter.Style.PRETTY; +import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR; +import static org.elasticsearch.rest.RestStatus.OK; + +import com.amazon.opendistroforelasticsearch.sql.common.response.ResponseListener; +import com.amazon.opendistroforelasticsearch.sql.common.setting.Settings; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.security.SecurityAccess; +import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.QueryResponse; +import com.amazon.opendistroforelasticsearch.sql.plugin.request.PPLQueryRequestFactory; +import com.amazon.opendistroforelasticsearch.sql.ppl.PPLService; +import com.amazon.opendistroforelasticsearch.sql.ppl.config.PPLServiceConfig; +import com.amazon.opendistroforelasticsearch.sql.protocol.response.QueryResult; +import com.amazon.opendistroforelasticsearch.sql.protocol.response.format.SimpleJsonResponseFormatter; +import java.io.IOException; +import java.security.PrivilegedExceptionAction; +import java.util.Collections; +import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.BytesRestResponse; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestStatus; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; + +public class RestPPLQueryAction extends BaseRestHandler { + public static final String QUERY_API_ENDPOINT = "/_opendistro/_ppl"; + + private static final Logger LOG = LogManager.getLogger(); + + /** + * Cluster service required by bean initialization. + */ + private final ClusterService clusterService; + + /** + * Settings required by been initialization. + */ + private final Settings pluginSettings; + + /** + * Constructor of RestPPLQueryAction. + */ + public RestPPLQueryAction(RestController restController, ClusterService clusterService, + Settings pluginSettings) { + super(); + this.clusterService = clusterService; + this.pluginSettings = pluginSettings; + } + + @Override + public List routes() { + return Collections.singletonList( + new Route(RestRequest.Method.POST, QUERY_API_ENDPOINT) + ); + } + + @Override + public String getName() { + return "ppl_query_action"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient nodeClient) { + PPLService pplService = createPPLService(nodeClient); + return channel -> pplService.execute( + PPLQueryRequestFactory.getPPLRequest(request), createListener(channel)); + } + + /** + * Ideally, the AnnotationConfigApplicationContext should be shared across Plugin. By default, + * spring construct all the bean as singleton. Currently, there are no better solution to + * create the bean in protocol scope. The limitations are + * alt-1, add annotation for bean @Scope(value = SCOPE_PROTOTYPE, proxyMode = TARGET_CLASS), it + * works by add the proxy, + * but when running in Elasticsearch, all the operation need security permission whic is hard + * to control. + * alt-2, using ObjectFactory with @Autowired, it also works, but require add to all the + * configuration. + * We will revisit the current solution if any major issue found. + */ + private PPLService createPPLService(NodeClient client) { + return doPrivileged(() -> { + AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); + context.registerBean(ClusterService.class, () -> clusterService); + context.registerBean(NodeClient.class, () -> client); + context.registerBean(Settings.class, () -> pluginSettings); + context.register(ElasticsearchPluginConfig.class); + context.register(PPLServiceConfig.class); + context.refresh(); + return context.getBean(PPLService.class); + }); + } + + private ResponseListener createListener(RestChannel channel) { + SimpleJsonResponseFormatter formatter = + new SimpleJsonResponseFormatter(PRETTY); // TODO: decide format and pretty from URL param + return new ResponseListener() { + @Override + public void onResponse(QueryResponse response) { + sendResponse(OK, formatter.format(new QueryResult(response.getResults()))); + } + + @Override + public void onFailure(Exception e) { + LOG.error("Error happened during query handling", e); + sendResponse(INTERNAL_SERVER_ERROR, formatter.format(e)); + } + + private void sendResponse(RestStatus status, String content) { + channel.sendResponse( + new BytesRestResponse(status, "application/json; charset=UTF-8", content)); + } + }; + } + + private T doPrivileged(PrivilegedExceptionAction action) { + try { + return SecurityAccess.doPrivileged(action); + } catch (IOException e) { + throw new IllegalStateException("Failed to perform privileged action", e); + } + } + +} diff --git a/plugin/src/main/plugin-metadata/plugin-security.policy b/plugin/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 0000000000..6b697ff1c0 --- /dev/null +++ b/plugin/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,21 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +grant { + // For Spring IOC + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + permission java.lang.RuntimePermission "accessDeclaredMembers"; + permission java.lang.RuntimePermission "defineClass"; +}; \ No newline at end of file diff --git a/ppl/build.gradle b/ppl/build.gradle new file mode 100644 index 0000000000..3818d805f4 --- /dev/null +++ b/ppl/build.gradle @@ -0,0 +1,74 @@ +plugins { + id 'java' + id "io.freefair.lombok" + id 'jacoco' + id 'antlr' +} + +repositories { + mavenCentral() +} + +generateGrammarSource { + arguments += ['-visitor', '-package', 'com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser'] + source = sourceSets.main.antlr + outputDirectory = file("build/generated-src/antlr/main/com/amazon/opendistroforelasticsearch/sql/ppl/antlr/parser") +} + +configurations { + compile { + extendsFrom = extendsFrom.findAll { it != configurations.antlr } + } +} + +dependencies { + antlr "org.antlr:antlr4:4.7.1" + + compile "org.antlr:antlr4-runtime:4.7.1" + compile group: 'com.google.guava', name: 'guava', version: '23.0' + compile group: 'org.elasticsearch', name: 'elasticsearch-x-content', version: "${es_version}" + compile group: 'org.json', name: 'json', version: '20180813' + compile group: 'org.springframework', name: 'spring-context', version: '5.2.5.RELEASE' + compile group: 'org.springframework', name: 'spring-beans', version: '5.2.5.RELEASE' + compile project(':common') + compile project(':core') + compile project(':protocol') + + testCompile group: 'junit', name: 'junit', version: '4.12' + testCompile group: 'org.mockito', name: 'mockito-core', version: '3.3.3' + +} + +jacoco { + toolVersion = "0.8.5" +} + +jacocoTestReport { + reports { + html.enabled true + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, + exclude: ['**/antlr/parser/**']) + })) + } +} +test.finalizedBy(project.tasks.jacocoTestReport) +jacocoTestCoverageVerification { + violationRules { + rule { + limit { + minimum = 1.0 + } + + } + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, + exclude: ['**/antlr/parser/**']) + })) + } +} +check.dependsOn jacocoTestCoverageVerification \ No newline at end of file diff --git a/ppl/lombok.config b/ppl/lombok.config new file mode 100644 index 0000000000..189c0bef98 --- /dev/null +++ b/ppl/lombok.config @@ -0,0 +1,3 @@ +# This file is generated by the 'io.freefair.lombok' Gradle plugin +config.stopBubbling = true +lombok.addLombokGeneratedAnnotation = true diff --git a/ppl/src/main/antlr/OpenDistroPPLLexer.g4 b/ppl/src/main/antlr/OpenDistroPPLLexer.g4 new file mode 100644 index 0000000000..8b5eb77e10 --- /dev/null +++ b/ppl/src/main/antlr/OpenDistroPPLLexer.g4 @@ -0,0 +1,261 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +lexer grammar OpenDistroPPLLexer; + +channels { WHITESPACE, ERRORCHANNEL } + + +// COMMAND KEYWORDS +SEARCH: 'SEARCH'; +FROM: 'FROM'; +WHERE: 'WHERE'; +FIELDS: 'FIELDS'; +RENAME: 'RENAME'; +STATS: 'STATS'; +DEDUP: 'DEDUP'; +SORT: 'SORT'; +EVAL: 'EVAL'; + +// COMMAND ASSIST KEYWORDS +AS: 'AS'; +BY: 'BY'; +SOURCE: 'SOURCE'; +INDEX: 'INDEX'; +D: 'D'; +DESC: 'DESC'; + +// CLAUSE KEYWORDS +SORTBY: 'SORTBY'; + +// FIELD KEYWORDS +AUTO: 'AUTO'; +STR: 'STR'; +IP: 'IP'; +NUM: 'NUM'; + +// ARGUMENT KEYWORDS +KEEPEMPTY: 'KEEPEMPTY'; +CONSECUTIVE: 'CONSECUTIVE'; +DEDUP_SPLITVALUES: 'DEDUP_SPLITVALUES'; +PARTITIONS: 'PARTITIONS'; +ALLNUM: 'ALLNUM'; +DELIM: 'DELIM'; + +// COMPARISON FUNCTION KEYWORDS +CASE: 'CASE'; +IN: 'IN'; + +// LOGICAL KEYWORDS +NOT: 'NOT'; +OR: 'OR'; +AND: 'AND'; +XOR: 'XOR'; +TRUE: 'TRUE'; +FALSE: 'FALSE'; +LIKE: 'LIKE'; + +// DATASET TYPES +DATAMODEL: 'DATAMODEL'; +LOOKUP: 'LOOKUP'; +SAVEDSEARCH: 'SAVEDSEARCH'; + +// SPECIAL CHARACTERS AND OPERATORS +PIPE: '|'; +COMMA: ','; +DOT: '.'; +EQUAL: '='; +GREATER: '>'; +LESS: '<'; +NOT_GREATER: '<' '='; +NOT_LESS: '>' '='; +NOT_EQUAL: '!' '='; +PLUS: '+'; +MINUS: '-'; +STAR: '*'; +DIVIDE: '/'; +MODULE: '%'; +EXCLAMATION_SYMBOL: '!'; +COLON: ':'; +LT_PRTHS: '('; +RT_PRTHS: ')'; +LT_SQR_PRTHS: '['; +RT_SQR_PRTHS: ']'; +SINGLE_QUOTE: '\''; +DOUBLE_QUOTE: '"'; +BACKTICK: '`'; + +// AGGREGATIONS +AVG: 'AVG'; +COUNT: 'COUNT'; +DISTINCT_COUNT: 'DISTINCT_COUNT'; +ESTDC: 'ESTDC'; +ESTDC_ERROR: 'ESTDC_ERROR'; +MAX: 'MAX'; +MEAN: 'MEAN'; +MEDIAN: 'MEDIAN'; +MIN: 'MIN'; +MODE: 'MODE'; +RANGE: 'RANGE'; +STDEV: 'STDEV'; +STDEVP: 'STDEVP'; +SUM: 'SUM'; +SUMSQ: 'SUMSQ'; +VAR: 'VAR'; +VARP: 'VARP'; +PERCENTILE: 'PERCENTILE'; +FIRST: 'FIRST'; +LAST: 'LAST'; +LIST: 'LIST'; +VALUES: 'VALUES'; +EARLIEST: 'EARLIEST'; +EARLIEST_TIME: 'EARLIEST_TIME'; +LATEST: 'LATEST'; +LATEST_TIME: 'LATEST_TIME'; +PER_DAY: 'PER_DAY'; +PER_HOUR: 'PER_HOUR'; +PER_MINUTE: 'PER_MINUTE'; +PER_SECOND: 'PER_SECOND'; +RATE: 'RATE'; +SPARKLINE: 'SPARKLINE'; +C: 'C'; +DC: 'DC'; + +// BASIC FUNCTIONS +ABS: 'ABS'; +ACOS: 'ACOS'; +ADD: 'ADD'; +ADDDATE: 'ADDDATE'; +ADDTIME: 'ADDTIME'; +ASCII: 'ASCII'; +ASIN: 'ASIN'; +ATAN: 'ATAN'; +ATAN2: 'ATAN2'; +CBRT: 'CBRT'; +CEIL: 'CEIL'; +CONCAT: 'CONCAT'; +CONCAT_WS: 'CONCAT_WS'; +COS: 'COS'; +COSH: 'COSH'; +COT: 'COT'; +CURDATE: 'CURDATE'; +DATE: 'DATE'; +DATE_FORMAT: 'DATE_FORMAT'; +DAYOFMONTH: 'DAYOFMONTH'; +DEGREES: 'DEGREES'; +E: 'E'; +EXP: 'EXP'; +EXPM1: 'EXPM1'; +FLOOR: 'FLOOR'; +GREATEST: 'GREATEST'; +IF: 'IF'; +IFNULL: 'IFNULL'; +ISNULL: 'ISNULL'; +LEAST: 'LEAST'; +LEFT: 'LEFT'; +LENGTH: 'LENGTH'; +LN: 'LN'; +LOCATE: 'LOCATE'; +LOG: 'LOG'; +LOG10: 'LOG10'; +LOG2: 'LOG2'; +LOWER: 'LOWER'; +LTRIM: 'LTRIM'; +MAKETIME: 'MAKETIME'; +MISSING: 'MISSING'; +MODULUS: 'MODULUS'; +MONTH: 'MONTH'; +MONTHNAME: 'MONTHNAME'; +MULTIPLY: 'MULTIPLY'; +NOW: 'NOW'; +PI: 'PI'; +POW: 'POW'; +POWER: 'POWER'; +RADIANS: 'RADIANS'; +RAND: 'RAND'; +REPLACE: 'REPLACE'; +RIGHT: 'RIGHT'; +RINT: 'RINT'; +ROUND: 'ROUND'; +RTRIM: 'RTRIM'; +SIGN: 'SIGN'; +SIGNUM: 'SIGNUM'; +SIN: 'SIN'; +SINH: 'SINH'; +SQRT: 'SQRT'; +SUBTRACT: 'SUBTRACT'; +SUBSTRING: 'SUBSTRING'; +TAN: 'TAN'; +TIMESTAMP: 'TIMESTAMP'; +TRIM: 'TRIM'; +UPPER: 'UPPER'; +YEAR: 'YEAR'; + +// ES FUNCTIONS +DATE_HISTOGRAM: 'DATE_HISTOGRAM'; +DAY_OF_MONTH: 'DAY_OF_MONTH'; +DAY_OF_YEAR: 'DAY_OF_YEAR'; +DAY_OF_WEEK: 'DAY_OF_WEEK'; +EXCLUDE: 'EXCLUDE'; +EXTENDED_STATS: 'EXTENDED_STATS'; +FIELD: 'FIELD'; +FILTER: 'FILTER'; +GEO_BOUNDING_BOX: 'GEO_BOUNDING_BOX'; +GEO_CELL: 'GEO_CELL'; +GEO_DISTANCE: 'GEO_DISTANCE'; +GEO_DISTANCE_RANGE: 'GEO_DISTANCE_RANGE'; +GEO_INTERSECTS: 'GEO_INTERSECTS'; +GEO_POLYGON: 'GEO_POLYGON'; +HISTOGRAM: 'HISTOGRAM'; +HOUR_OF_DAY: 'HOUR_OF_DAY'; +INCLUDE: 'INCLUDE'; +IN_TERMS: 'IN_TERMS'; +MATCHPHRASE: 'MATCHPHRASE'; +MATCH_PHRASE: 'MATCH_PHRASE'; +MATCHQUERY: 'MATCHQUERY'; +MATCH_QUERY: 'MATCH_QUERY'; +MINUTE_OF_DAY: 'MINUTE_OF_DAY'; +MINUTE_OF_HOUR: 'MINUTE_OF_HOUR'; +MONTH_OF_YEAR: 'MONTH_OF_YEAR'; +MULTIMATCH: 'MULTIMATCH'; +MULTI_MATCH: 'MULTI_MATCH'; +NESTED: 'NESTED'; +PERCENTILES: 'PERCENTILES'; +REGEXP_QUERY: 'REGEXP_QUERY'; +REVERSE_NESTED: 'REVERSE_NESTED'; +QUERY: 'QUERY'; +SCORE: 'SCORE'; +SECOND_OF_MINUTE: 'SECOND_OF_MINUTE'; +TERM: 'TERM'; +TERMS: 'TERMS'; +TOPHITS: 'TOPHITS'; +WEEK_OF_YEAR: 'WEEK_OF_YEAR'; +WILDCARDQUERY: 'WILDCARDQUERY'; +WILDCARD_QUERY: 'WILDCARD_QUERY'; + +// LITERALS AND VALUES +//STRING_LITERAL: DQUOTA_STRING | SQUOTA_STRING | BQUOTA_STRING; +ID: ID_LITERAL; +INTEGER_LITERAL: DEC_DIGIT+; +DECIMAL_LITERAL: (DEC_DIGIT+)? '.' DEC_DIGIT+; + +fragment ID_LITERAL: [A-Z_$0-9@]*?[A-Z_$\-]+?[A-Z_$\-0-9]*; +DQUOTA_STRING: '"' ( '\\'. | '""' | ~('"'| '\\') )* '"'; +SQUOTA_STRING: '\'' ('\\'. | '\'\'' | ~('\'' | '\\'))* '\''; +BQUOTA_STRING: '`' ( '\\'. | '``' | ~('`'|'\\'))* '`'; +fragment DEC_DIGIT: [0-9]; + + +ERROR_RECOGNITION: . -> channel(ERRORCHANNEL); \ No newline at end of file diff --git a/ppl/src/main/antlr/OpenDistroPPLParser.g4 b/ppl/src/main/antlr/OpenDistroPPLParser.g4 new file mode 100644 index 0000000000..f9e3cdb7e3 --- /dev/null +++ b/ppl/src/main/antlr/OpenDistroPPLParser.g4 @@ -0,0 +1,273 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + + +parser grammar OpenDistroPPLParser; +options { tokenVocab=OpenDistroPPLLexer; } + +root + : pplStatement? EOF + ; + +/** statement */ +pplStatement + : searchCommand (PIPE commands)* + ; + +/** commands */ +commands + : whereCommand | fieldsCommand | renameCommand | statsCommand | dedupCommand | sortCommand | evalCommand + ; + +searchCommand + : (SEARCH)? fromClause #searchFrom + | (SEARCH)? fromClause logicalExpression #searchFromFilter + | (SEARCH)? logicalExpression fromClause #searchFilterFrom + ; + +whereCommand + : WHERE logicalExpression + ; + +fieldsCommand + : FIELDS (PLUS | MINUS)? wcFieldList + ; + +renameCommand + : RENAME renameClasue (COMMA renameClasue)* + ; + +statsCommand + : STATS + (PARTITIONS EQUAL partitions=integerLiteral)? + (ALLNUM EQUAL allnum=booleanLiteral)? + (DELIM EQUAL delim=stringLiteral)? + statsAggTerm (COMMA statsAggTerm)* + (byClause)? + (DEDUP_SPLITVALUES EQUAL dedupsplit=booleanLiteral)? + ; + +dedupCommand + : DEDUP + (number=integerLiteral)? + fieldList + (KEEPEMPTY EQUAL keepempty=booleanLiteral)? + (CONSECUTIVE EQUAL consecutive=booleanLiteral)? + ; + +sortCommand + : SORT (count=integerLiteral)? sortbyClause (D | DESC)? + ; + +evalCommand + : EVAL evalClause (COMMA evalClause)* + ; + +/** clauses */ +fromClause + : SOURCE EQUAL tableSource + | INDEX EQUAL tableSource + ; + +renameClasue + : orignalField=wcFieldExpression AS renamedField=wcFieldExpression + ; + +byClause + : BY fieldList + ; + +sortbyClause + : sortField (COMMA sortField)* + ; + +evalClause + : fieldExpression EQUAL expression + ; + +/** aggregation terms */ +statsAggTerm + : statsFunction (AS alias=wcFieldExpression)? + ; + +/** aggregation functions */ +statsFunction + : statsFunctionName LT_PRTHS valueExpression RT_PRTHS #statsFunctionCall + | percentileAggFunction #percentileAggFunctionCall + ; + +statsFunctionName + : AVG | COUNT | SUM + ; + +percentileAggFunction + : PERCENTILE '<' value=integerLiteral '>' LT_PRTHS aggField=fieldExpression RT_PRTHS + ; + +/** expressions */ +expression + : logicalExpression + | comparisonExpression + | valueExpression + ; + +logicalExpression + : comparisonExpression #comparsion + | NOT logicalExpression #logicalNot + | left=logicalExpression OR right=logicalExpression #logicalOr + | left=logicalExpression (AND)? right=logicalExpression #logicalAnd + | left=logicalExpression XOR right=logicalExpression #logicalXor + ; + +comparisonExpression + : left=valueExpression comparisonOperator right=valueExpression #compareExpr + | valueExpression IN valueList #inExpr + ; + +valueExpression + : left=valueExpression binaryOperator right=valueExpression #binaryArithmetic + | LT_PRTHS left=valueExpression binaryOperator + right=valueExpression RT_PRTHS #parentheticBinaryArithmetic + | primaryExpression #valueExpressionDefault + ; + +primaryExpression + : evalFunctionCall + | fieldExpression + | literalValue + ; + +/** tables */ +tableSource + : qualifiedName + ; + +/** fields */ +fieldList + : fieldExpression (COMMA fieldExpression)* + ; + +wcFieldList + : wcFieldExpression (COMMA wcFieldExpression)* + ; + +sortField + : (PLUS | MINUS)? sortFieldExpression + ; + +sortFieldExpression + : fieldExpression + | AUTO LT_PRTHS fieldExpression RT_PRTHS + | STR LT_PRTHS fieldExpression RT_PRTHS + | IP LT_PRTHS fieldExpression RT_PRTHS + | NUM LT_PRTHS fieldExpression RT_PRTHS + ; + +fieldExpression + : qualifiedName + ; + +wcFieldExpression + : wcQualifiedName + ; + +/** functions */ +evalFunctionCall + : evalFunctionName LT_PRTHS functionArgs RT_PRTHS + ; + +evalFunctionName + : mathematicalFunctionBase + | dateAndTimeFunctionBase + | textFunctionBase + ; + +functionArgs + : functionArg (COMMA functionArg)* + ; + +functionArg + : valueExpression + ; + +mathematicalFunctionBase + : ABS + ; + +dateAndTimeFunctionBase + : + ; + +textFunctionBase + : + ; + +/** operators */ +comparisonOperator + : EQUAL | NOT_EQUAL | LESS | NOT_LESS | GREATER | NOT_GREATER | LIKE + ; + +binaryOperator + : PLUS | MINUS | STAR | DIVIDE | MODULE + ; + +/** literals and values*/ +literalValue + : stringLiteral + | (PLUS | MINUS)? integerLiteral + | (PLUS | MINUS)? decimalLiteral + | booleanLiteral + ; + +stringLiteral + : DQUOTA_STRING | SQUOTA_STRING + ; + +integerLiteral + : INTEGER_LITERAL + ; + +decimalLiteral + : DECIMAL_LITERAL + ; + +booleanLiteral + : TRUE | FALSE + ; + +valueList + : LT_PRTHS literalValue (COMMA literalValue)* RT_PRTHS + ; + +qualifiedName + : ident (DOT ident)* + ; + +wcQualifiedName + : wildcard (DOT wildcard)* + ; + +ident + : (DOT)? ID + | BACKTICK ident BACKTICK + | BQUOTA_STRING + ; + +wildcard + : ident (MODULE ident)* (MODULE)? + | SINGLE_QUOTE wildcard SINGLE_QUOTE + | DOUBLE_QUOTE wildcard DOUBLE_QUOTE + | BACKTICK wildcard BACKTICK + ; diff --git a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/PPLService.java b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/PPLService.java new file mode 100644 index 0000000000..0c3e437c68 --- /dev/null +++ b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/PPLService.java @@ -0,0 +1,69 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl; + +import static com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.QueryResponse; + +import com.amazon.opendistroforelasticsearch.sql.analysis.AnalysisContext; +import com.amazon.opendistroforelasticsearch.sql.analysis.Analyzer; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; +import com.amazon.opendistroforelasticsearch.sql.common.response.ResponseListener; +import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine; +import com.amazon.opendistroforelasticsearch.sql.planner.Planner; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import com.amazon.opendistroforelasticsearch.sql.ppl.antlr.PPLSyntaxParser; +import com.amazon.opendistroforelasticsearch.sql.ppl.domain.PPLQueryRequest; +import com.amazon.opendistroforelasticsearch.sql.ppl.parser.AstBuilder; +import com.amazon.opendistroforelasticsearch.sql.ppl.parser.AstExpressionBuilder; +import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; +import lombok.RequiredArgsConstructor; +import org.antlr.v4.runtime.tree.ParseTree; + +@RequiredArgsConstructor +public class PPLService { + private final PPLSyntaxParser parser; + + private final Analyzer analyzer; + + private final StorageEngine storageEngine; + + private final ExecutionEngine executionEngine; + + /** + * Execute the {@link PPLQueryRequest}, using {@link ResponseListener} to get response. + * @param request {@link PPLQueryRequest} + * @param listener {@link ResponseListener} + */ + public void execute(PPLQueryRequest request, ResponseListener listener) { + try { + // 1.Parse query and convert parse tree (CST) to abstract syntax tree (AST) + ParseTree cst = parser.analyzeSyntax(request.getRequest()); + UnresolvedPlan ast = cst.accept(new AstBuilder(new AstExpressionBuilder())); + + // 2.Analyze abstract syntax to generate logical plan + LogicalPlan logicalPlan = analyzer.analyze(ast, new AnalysisContext()); + + // 3.Generate optimal physical plan from logical plan + PhysicalPlan physicalPlan = new Planner(storageEngine).plan(logicalPlan); + + // 4.Execute physical plan and send response + executionEngine.execute(physicalPlan, listener); + } catch (Exception e) { + listener.onFailure(e); + } + } +} diff --git a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/antlr/PPLSyntaxParser.java b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/antlr/PPLSyntaxParser.java new file mode 100644 index 0000000000..c638ae7ab1 --- /dev/null +++ b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/antlr/PPLSyntaxParser.java @@ -0,0 +1,48 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl.antlr; + +import com.amazon.opendistroforelasticsearch.sql.common.antlr.CaseInsensitiveCharStream; +import com.amazon.opendistroforelasticsearch.sql.common.antlr.SyntaxAnalysisErrorListener; +import com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLLexer; +import com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.Lexer; +import org.antlr.v4.runtime.tree.ParseTree; + +/** + * PPL Syntax Parser. + */ +public class PPLSyntaxParser { + /** + * Analyze the query syntax. + */ + public ParseTree analyzeSyntax(String query) { + OpenDistroPPLParser parser = createParser(createLexer(query)); + parser.addErrorListener(new SyntaxAnalysisErrorListener()); + return parser.root(); + } + + private OpenDistroPPLParser createParser(Lexer lexer) { + return new OpenDistroPPLParser( + new CommonTokenStream(lexer)); + } + + private OpenDistroPPLLexer createLexer(String query) { + return new OpenDistroPPLLexer( + new CaseInsensitiveCharStream(query)); + } +} diff --git a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/config/PPLServiceConfig.java b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/config/PPLServiceConfig.java new file mode 100644 index 0000000000..f1a6e3aae0 --- /dev/null +++ b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/config/PPLServiceConfig.java @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl.config; + +import com.amazon.opendistroforelasticsearch.sql.analysis.Analyzer; +import com.amazon.opendistroforelasticsearch.sql.analysis.ExpressionAnalyzer; +import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine; +import com.amazon.opendistroforelasticsearch.sql.expression.config.ExpressionConfig; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository; +import com.amazon.opendistroforelasticsearch.sql.ppl.PPLService; +import com.amazon.opendistroforelasticsearch.sql.ppl.antlr.PPLSyntaxParser; +import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +@Configuration +@Import({ExpressionConfig.class}) +public class PPLServiceConfig { + + @Autowired + private StorageEngine storageEngine; + + @Autowired + private ExecutionEngine executionEngine; + + @Autowired + private BuiltinFunctionRepository functionRepository; + + @Bean + public Analyzer analyzer() { + return new Analyzer(new ExpressionAnalyzer(functionRepository), storageEngine); + } + + @Bean + public PPLService pplService() { + return new PPLService(new PPLSyntaxParser(), analyzer(), storageEngine, executionEngine); + } + +} diff --git a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/domain/PPLQueryRequest.java b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/domain/PPLQueryRequest.java new file mode 100644 index 0000000000..9bb348fd43 --- /dev/null +++ b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/domain/PPLQueryRequest.java @@ -0,0 +1,31 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl.domain; + +import lombok.RequiredArgsConstructor; +import org.json.JSONObject; + +@RequiredArgsConstructor +public class PPLQueryRequest { + public static final PPLQueryRequest NULL = new PPLQueryRequest("", null); + + private final String pplQuery; + private final JSONObject jsonContent; + + public String getRequest() { + return pplQuery; + } +} diff --git a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/domain/PPLQueryResponse.java b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/domain/PPLQueryResponse.java new file mode 100644 index 0000000000..ff7a09581a --- /dev/null +++ b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/domain/PPLQueryResponse.java @@ -0,0 +1,19 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl.domain; + +public class PPLQueryResponse { +} diff --git a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstBuilder.java new file mode 100644 index 0000000000..9924f3ac12 --- /dev/null +++ b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstBuilder.java @@ -0,0 +1,226 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl.parser; + +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.DedupCommandContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.EvalCommandContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.FieldsCommandContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.FromClauseContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.PplStatementContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.RenameCommandContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.SearchFilterFromContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.SearchFromContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.SearchFromFilterContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.SortCommandContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.StatsCommandContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.WhereCommandContext; + +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Field; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Let; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Map; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Aggregation; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Dedupe; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Eval; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Filter; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Project; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Relation; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Rename; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; +import com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser; +import com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParserBaseVisitor; +import com.amazon.opendistroforelasticsearch.sql.ppl.utils.ArgumentFactory; +import com.google.common.collect.ImmutableList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import org.antlr.v4.runtime.tree.ParseTree; + +/** + * Class of building the AST. + * Refines the visit path and build the AST nodes + */ +@RequiredArgsConstructor +public class AstBuilder extends OpenDistroPPLParserBaseVisitor { + private final AstExpressionBuilder expressionBuilder; + + @Override + public UnresolvedPlan visitPplStatement(PplStatementContext ctx) { + UnresolvedPlan search = visit(ctx.searchCommand()); + return ctx.commands() + .stream() + .map(this::visit) + .reduce(search, (r, e) -> e.attach(r)); + } + + /** + * Search command. + */ + @Override + public UnresolvedPlan visitSearchFrom(SearchFromContext ctx) { + return visitFromClause(ctx.fromClause()); + } + + @Override + public UnresolvedPlan visitSearchFromFilter(SearchFromFilterContext ctx) { + return new Filter(visitExpression(ctx.logicalExpression())).attach(visit(ctx.fromClause())); + } + + @Override + public UnresolvedPlan visitSearchFilterFrom(SearchFilterFromContext ctx) { + return new Filter(visitExpression(ctx.logicalExpression())).attach(visit(ctx.fromClause())); + } + + /** + * Where command. + */ + @Override + public UnresolvedPlan visitWhereCommand(WhereCommandContext ctx) { + return new Filter(visitExpression(ctx.logicalExpression())); + } + + /** + * Fields command. + */ + @Override + public UnresolvedPlan visitFieldsCommand(FieldsCommandContext ctx) { + return new Project( + ctx.wcFieldList() + .wcFieldExpression() + .stream() + .map(this::visitExpression) + .collect(Collectors.toList()), + ArgumentFactory.getArgumentList(ctx) + ); + } + + /** + * Rename command. + */ + @Override + public UnresolvedPlan visitRenameCommand(RenameCommandContext ctx) { + return new Rename( + ctx.renameClasue() + .stream() + .map(ct -> new Map(visitExpression(ct.orignalField), visitExpression(ct.renamedField))) + .collect(Collectors.toList()) + ); + } + + /** + * Stats command. + */ + @Override + public UnresolvedPlan visitStatsCommand(StatsCommandContext ctx) { + ImmutableList.Builder aggListBuilder = new ImmutableList.Builder<>(); + ImmutableList.Builder renameListBuilder = new ImmutableList.Builder<>(); + for (OpenDistroPPLParser.StatsAggTermContext aggCtx : ctx.statsAggTerm()) { + UnresolvedExpression aggExpression = visitExpression(aggCtx.statsFunction()); + aggListBuilder.add(aggExpression); + if (aggCtx.alias != null) { + renameListBuilder + .add(new Map(aggExpression, visitExpression(aggCtx.alias))); + } + } + List groupList = ctx.byClause() == null ? Collections.emptyList() : + ctx.byClause() + .fieldList() + .fieldExpression() + .stream() + .map(this::visitExpression) + .collect(Collectors.toList()); + Aggregation aggregation = new Aggregation( + aggListBuilder.build(), + Collections.emptyList(), + groupList, + ArgumentFactory.getArgumentList(ctx) + ); + List renameList = renameListBuilder.build(); + return renameList.isEmpty() ? aggregation : new Rename(renameList, aggregation); + } + + /** + * Dedup command. + */ + @Override + public UnresolvedPlan visitDedupCommand(DedupCommandContext ctx) { + return new Dedupe( + ArgumentFactory.getArgumentList(ctx), + ctx.fieldList() + .fieldExpression() + .stream() + .map(field -> (Field) visitExpression(field)) + .collect(Collectors.toList()) + ); + } + + /** + * Sort command. + */ + @Override + public UnresolvedPlan visitSortCommand(SortCommandContext ctx) { + return new Sort( + ArgumentFactory.getArgumentList(ctx), + ctx.sortbyClause() + .sortField() + .stream() + .map(sort -> (Field) visitExpression(sort)) + .collect(Collectors.toList()) + ); + } + + /** + * Eval command. + */ + @Override + public UnresolvedPlan visitEvalCommand(EvalCommandContext ctx) { + return new Eval( + ctx.evalClause() + .stream() + .map(ct -> (Let) visitExpression(ct)) + .collect(Collectors.toList()) + ); + } + + /** + * From clause. + */ + @Override + public UnresolvedPlan visitFromClause(FromClauseContext ctx) { + return new Relation(visitExpression(ctx.tableSource().qualifiedName())); + } + + /** + * Navigate to & build AST expression. + */ + private UnresolvedExpression visitExpression(ParseTree tree) { + return expressionBuilder.visit(tree); + } + + /** + * Simply return non-default value for now. + */ + @Override + protected UnresolvedPlan aggregateResult(UnresolvedPlan aggregate, UnresolvedPlan nextResult) { + if (nextResult != defaultResult()) { + return nextResult; + } + return aggregate; + } + +} diff --git a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilder.java new file mode 100644 index 0000000000..f8c4b351e0 --- /dev/null +++ b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilder.java @@ -0,0 +1,232 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl.parser; + +import static com.amazon.opendistroforelasticsearch.sql.common.utils.StringUtils.unquoteIdentifier; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.BinaryArithmeticContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.BooleanLiteralContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.CompareExprContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.DecimalLiteralContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.EvalClauseContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.EvalFunctionCallContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.FieldExpressionContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.InExprContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.IntegerLiteralContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.LogicalAndContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.LogicalNotContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.LogicalOrContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.LogicalXorContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.ParentheticBinaryArithmeticContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.PercentileAggFunctionContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.QualifiedNameContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.SortFieldContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.StatsFunctionCallContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.StringLiteralContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.WcFieldExpressionContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.WcQualifiedNameContext; + +import com.amazon.opendistroforelasticsearch.sql.ast.expression.AggregateFunction; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.And; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Argument; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Compare; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.DataType; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Field; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Function; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.In; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Let; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Literal; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Not; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Or; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.QualifiedName; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Xor; +import com.amazon.opendistroforelasticsearch.sql.common.utils.StringUtils; +import com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParserBaseVisitor; +import com.amazon.opendistroforelasticsearch.sql.ppl.utils.ArgumentFactory; +import java.util.Arrays; +import java.util.Collections; +import java.util.stream.Collectors; +import org.antlr.v4.runtime.ParserRuleContext; + +/** + * Class of building AST Expression nodes. + */ +public class AstExpressionBuilder extends OpenDistroPPLParserBaseVisitor { + /** + * Eval clause. + */ + @Override + public UnresolvedExpression visitEvalClause(EvalClauseContext ctx) { + return new Let((Field) visit(ctx.fieldExpression()), visit(ctx.expression())); + } + + /** + * Logical expression excluding boolean, comparison. + */ + @Override + public UnresolvedExpression visitLogicalNot(LogicalNotContext ctx) { + return new Not(visit(ctx.logicalExpression())); + } + + @Override + public UnresolvedExpression visitLogicalOr(LogicalOrContext ctx) { + return new Or(visit(ctx.left), visit(ctx.right)); + } + + @Override + public UnresolvedExpression visitLogicalAnd(LogicalAndContext ctx) { + return new And(visit(ctx.left), visit(ctx.right)); + } + + @Override + public UnresolvedExpression visitLogicalXor(LogicalXorContext ctx) { + return new Xor(visit(ctx.left), visit(ctx.right)); + } + + /** + * Comparison expression. + */ + @Override + public UnresolvedExpression visitCompareExpr(CompareExprContext ctx) { + return new Compare(ctx.comparisonOperator().getText(), visit(ctx.left), visit(ctx.right)); + } + + @Override + public UnresolvedExpression visitInExpr(InExprContext ctx) { + return new In( + visit(ctx.valueExpression()), + ctx.valueList() + .literalValue() + .stream() + .map(this::visitLiteralValue) + .collect(Collectors.toList())); + } + + /** + * Value Expression. + */ + @Override + public UnresolvedExpression visitBinaryArithmetic(BinaryArithmeticContext ctx) { + return new Function( + ctx.binaryOperator().getText(), + Arrays.asList(visit(ctx.left), visit(ctx.right)) + ); + } + + @Override + public UnresolvedExpression visitParentheticBinaryArithmetic( + ParentheticBinaryArithmeticContext ctx) { + return new Function( + ctx.binaryOperator().getText(), + Arrays.asList(visit(ctx.left), visit(ctx.right)) + ); + } + + /** + * Field expression. + */ + @Override + public UnresolvedExpression visitFieldExpression(FieldExpressionContext ctx) { + return new Field((QualifiedName) visit(ctx.qualifiedName())); + } + + @Override + public UnresolvedExpression visitWcFieldExpression(WcFieldExpressionContext ctx) { + return new Field((QualifiedName) visit(ctx.wcQualifiedName())); + } + + @Override + public UnresolvedExpression visitSortField(SortFieldContext ctx) { + return new Field( + ctx.sortFieldExpression().fieldExpression().getText(), + ArgumentFactory.getArgumentList(ctx) + ); + } + + /** + * Aggregation function. + */ + @Override + public UnresolvedExpression visitStatsFunctionCall(StatsFunctionCallContext ctx) { + return new AggregateFunction(ctx.statsFunctionName().getText(), visit(ctx.valueExpression())); + } + + @Override + public UnresolvedExpression visitPercentileAggFunction(PercentileAggFunctionContext ctx) { + return new AggregateFunction(ctx.PERCENTILE().getText(), visit(ctx.aggField), + Collections.singletonList(new Argument("rank", (Literal) visit(ctx.value)))); + } + + /** + * Eval function. + */ + @Override + public UnresolvedExpression visitEvalFunctionCall(EvalFunctionCallContext ctx) { + return new Function( + ctx.evalFunctionName().getText(), + ctx.functionArgs() + .functionArg() + .stream() + .map(this::visitFunctionArg) + .collect(Collectors.toList())); + } + + /** + * Literal and value. + */ + @Override + public UnresolvedExpression visitQualifiedName(QualifiedNameContext ctx) { + return new QualifiedName( + ctx.ident() + .stream() + .map(ParserRuleContext::getText) + .map(StringUtils::unquoteIdentifier) + .collect(Collectors.toList()) + ); + } + + @Override + public UnresolvedExpression visitWcQualifiedName(WcQualifiedNameContext ctx) { + return new QualifiedName( + ctx.wildcard() + .stream() + .map(ParserRuleContext::getText) + .map(StringUtils::unquoteIdentifier) + .collect(Collectors.toList()) + ); + } + + @Override + public UnresolvedExpression visitStringLiteral(StringLiteralContext ctx) { + return new Literal(unquoteIdentifier(ctx.getText()), DataType.STRING); + } + + @Override + public UnresolvedExpression visitIntegerLiteral(IntegerLiteralContext ctx) { + return new Literal(Integer.valueOf(ctx.getText()), DataType.INTEGER); + } + + @Override + public UnresolvedExpression visitDecimalLiteral(DecimalLiteralContext ctx) { + return new Literal(Double.valueOf(ctx.getText()), DataType.DOUBLE); + } + + @Override + public UnresolvedExpression visitBooleanLiteral(BooleanLiteralContext ctx) { + return new Literal(Boolean.valueOf(ctx.getText()), DataType.BOOLEAN); + } + +} diff --git a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/ArgumentFactory.java b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/ArgumentFactory.java new file mode 100644 index 0000000000..aaa839d0b2 --- /dev/null +++ b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/ArgumentFactory.java @@ -0,0 +1,146 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl.utils; + +import static com.amazon.opendistroforelasticsearch.sql.common.utils.StringUtils.unquoteIdentifier; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.BooleanLiteralContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.DedupCommandContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.FieldsCommandContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.IntegerLiteralContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.SortCommandContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.SortFieldContext; +import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.StatsCommandContext; + +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Argument; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.DataType; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Literal; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.antlr.v4.runtime.ParserRuleContext; + + +/** + * Util class to get all arguments as a list from the PPL command. + */ +public class ArgumentFactory { + + /** + * Get list of {@link Argument}. + * + * @param ctx FieldsCommandContext instance + * @return the list of arguments fetched from the fields command + */ + public static List getArgumentList(FieldsCommandContext ctx) { + return Collections.singletonList( + ctx.MINUS() != null + ? new Argument("exclude", new Literal(true, DataType.BOOLEAN)) + : new Argument("exclude", new Literal(false, DataType.BOOLEAN)) + ); + } + + /** + * Get list of {@link Argument}. + * + * @param ctx StatsCommandContext instance + * @return the list of arguments fetched from the stats command + */ + public static List getArgumentList(StatsCommandContext ctx) { + return Arrays.asList( + ctx.partitions != null + ? new Argument("partitions", getArgumentValue(ctx.partitions)) + : new Argument("partitions", new Literal(1, DataType.INTEGER)), + ctx.allnum != null + ? new Argument("allnum", getArgumentValue(ctx.allnum)) + : new Argument("allnum", new Literal(false, DataType.BOOLEAN)), + ctx.delim != null + ? new Argument("delim", getArgumentValue(ctx.delim)) + : new Argument("delim", new Literal(" ", DataType.STRING)), + ctx.dedupsplit != null + ? new Argument("dedupsplit", getArgumentValue(ctx.dedupsplit)) + : new Argument("dedupsplit", new Literal(false, DataType.BOOLEAN)) + ); + } + + /** + * Get list of {@link Argument}. + * + * @param ctx DedupCommandContext instance + * @return the list of arguments fetched from the dedup command + */ + public static List getArgumentList(DedupCommandContext ctx) { + return Arrays.asList( + ctx.number != null + ? new Argument("number", getArgumentValue(ctx.number)) + : new Argument("number", new Literal(1, DataType.INTEGER)), + ctx.keepempty != null + ? new Argument("keepempty", getArgumentValue(ctx.keepempty)) + : new Argument("keepempty", new Literal(false, DataType.BOOLEAN)), + ctx.consecutive != null + ? new Argument("consecutive", getArgumentValue(ctx.consecutive)) + : new Argument("consecutive", new Literal(false, DataType.BOOLEAN)) + ); + } + + /** + * Get list of {@link Argument}. + * + * @param ctx SortCommandContext instance + * @return the list of arguments fetched from the sort command + */ + public static List getArgumentList(SortCommandContext ctx) { + return Arrays.asList( + ctx.count != null + ? new Argument("count", getArgumentValue(ctx.count)) + : new Argument("count", new Literal(1000, DataType.INTEGER)), + ctx.D() != null || ctx.DESC() != null + ? new Argument("desc", new Literal(true, DataType.BOOLEAN)) + : new Argument("desc", new Literal(false, DataType.BOOLEAN)) + ); + } + + /** + * Get list of {@link Argument}. + * + * @param ctx SortFieldContext instance + * @return the list of arguments fetched from the sort field in sort command + */ + public static List getArgumentList(SortFieldContext ctx) { + return Arrays.asList( + ctx.MINUS() != null + ? new Argument("asc", new Literal(false, DataType.BOOLEAN)) + : new Argument("asc", new Literal(true, DataType.BOOLEAN)), + ctx.sortFieldExpression().AUTO() != null + ? new Argument("type", new Literal("auto", DataType.STRING)) + : ctx.sortFieldExpression().IP() != null + ? new Argument("type", new Literal("ip", DataType.STRING)) + : ctx.sortFieldExpression().NUM() != null + ? new Argument("type", new Literal("num", DataType.STRING)) + : ctx.sortFieldExpression().STR() != null + ? new Argument("type", new Literal("str", DataType.STRING)) + : new Argument("type", new Literal(null, DataType.NULL)) + ); + } + + private static Literal getArgumentValue(ParserRuleContext ctx) { + return ctx instanceof IntegerLiteralContext + ? new Literal(Integer.parseInt(ctx.getText()), DataType.INTEGER) + : ctx instanceof BooleanLiteralContext + ? new Literal(Boolean.valueOf(ctx.getText()), DataType.BOOLEAN) + : new Literal(unquoteIdentifier(ctx.getText()), DataType.STRING); + } + +} diff --git a/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/PPLServiceTest.java b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/PPLServiceTest.java new file mode 100644 index 0000000000..18df192acc --- /dev/null +++ b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/PPLServiceTest.java @@ -0,0 +1,111 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.common.response.ResponseListener; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine; +import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.QueryResponse; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import com.amazon.opendistroforelasticsearch.sql.ppl.config.PPLServiceConfig; +import com.amazon.opendistroforelasticsearch.sql.ppl.domain.PPLQueryRequest; +import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; +import com.amazon.opendistroforelasticsearch.sql.storage.Table; +import com.google.common.collect.ImmutableMap; +import java.util.Collections; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; + +@RunWith(MockitoJUnitRunner.class) +public class PPLServiceTest { + private AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); + + private PPLService pplService; + + @Mock + private StorageEngine storageEngine; + + @Mock + private ExecutionEngine executionEngine; + + @Mock + private Table table; + + @Mock + private PhysicalPlan plan; + + /** + * Setup the test context. + */ + @Before + public void setUp() { + when(table.getFieldTypes()).thenReturn(ImmutableMap.of("a", ExprCoreType.INTEGER)); + when(table.implement(any())).thenReturn(plan); + when(storageEngine.getTable(any())).thenReturn(table); + + context.registerBean(StorageEngine.class, () -> storageEngine); + context.registerBean(ExecutionEngine.class, () -> executionEngine); + context.register(PPLServiceConfig.class); + context.refresh(); + pplService = context.getBean(PPLService.class); + } + + @Test + public void testExecuteShouldPass() { + doAnswer(invocation -> { + ResponseListener listener = invocation.getArgument(1); + listener.onResponse(new QueryResponse(Collections.emptyList())); + return null; + }).when(executionEngine).execute(any(), any()); + + pplService.execute(new PPLQueryRequest("search source=t a=1", null), + new ResponseListener() { + @Override + public void onResponse(QueryResponse pplQueryResponse) { + + } + + @Override + public void onFailure(Exception e) { + Assert.fail(); + } + }); + } + + @Test + public void testExecuteWithIllegalQueryShouldBeCaughtByHandler() { + pplService.execute(new PPLQueryRequest("search", null), new ResponseListener() { + @Override + public void onResponse(QueryResponse pplQueryResponse) { + Assert.fail(); + } + + @Override + public void onFailure(Exception e) { + + } + }); + } +} \ No newline at end of file diff --git a/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/antlr/PPLSyntaxParserTest.java b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/antlr/PPLSyntaxParserTest.java new file mode 100644 index 0000000000..a847181847 --- /dev/null +++ b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/antlr/PPLSyntaxParserTest.java @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl.antlr; + +import static org.junit.Assert.assertNotEquals; + +import org.antlr.v4.runtime.tree.ParseTree; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +public class PPLSyntaxParserTest { + @Rule + public ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void testSearchCommandShouldPass() { + ParseTree tree = new PPLSyntaxParser().analyzeSyntax("search source=t a=1 b=2"); + assertNotEquals(null, tree); + } + + @Test + public void testSearchCommandIgnoreSearchKeywordShouldPass() { + ParseTree tree = new PPLSyntaxParser().analyzeSyntax("source=t a=1 b=2"); + assertNotEquals(null, tree); + } + + @Test + public void testSearchFieldsCommandShouldPass() { + ParseTree tree = new PPLSyntaxParser().analyzeSyntax("search source=t a=1 b=2 | fields a,b"); + assertNotEquals(null, tree); + } + + @Test + public void testSearchCommandWithoutSourceShouldFail() { + exceptionRule.expect(RuntimeException.class); + exceptionRule.expectMessage("Failed to parse query due to offending symbol"); + + new PPLSyntaxParser().analyzeSyntax("search a=1"); + } +} \ No newline at end of file diff --git a/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/config/PPLServiceConfigTest.java b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/config/PPLServiceConfigTest.java new file mode 100644 index 0000000000..15a8d9f537 --- /dev/null +++ b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/config/PPLServiceConfigTest.java @@ -0,0 +1,30 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl.config; + +import static org.junit.Assert.assertNotNull; + +import com.amazon.opendistroforelasticsearch.sql.ppl.PPLService; +import org.junit.Test; + +public class PPLServiceConfigTest { + @Test + public void testConfigPPLServiceShouldPass() { + PPLServiceConfig config = new PPLServiceConfig(); + PPLService service = config.pplService(); + assertNotNull(service); + } +} diff --git a/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/domain/PPLQueryRequestTest.java b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/domain/PPLQueryRequestTest.java new file mode 100644 index 0000000000..0b1e959d22 --- /dev/null +++ b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/domain/PPLQueryRequestTest.java @@ -0,0 +1,26 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl.domain; + +import org.junit.Test; + +public class PPLQueryRequestTest { + @Test + public void getRequestShouldPass() { + PPLQueryRequest request = new PPLQueryRequest("source=t a=1", null); + request.getRequest(); + } +} diff --git a/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/domain/PPLQueryResponseTest.java b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/domain/PPLQueryResponseTest.java new file mode 100644 index 0000000000..4a721d1aff --- /dev/null +++ b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/domain/PPLQueryResponseTest.java @@ -0,0 +1,25 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl.domain; + +import org.junit.Test; + +public class PPLQueryResponseTest { + @Test + public void testQueryResponse() { + PPLQueryResponse response = new PPLQueryResponse(); + } +} diff --git a/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstBuilderTest.java new file mode 100644 index 0000000000..d021dd5c68 --- /dev/null +++ b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstBuilderTest.java @@ -0,0 +1,317 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl.parser; + +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.agg; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.aggregate; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.argument; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.booleanLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.compare; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.dedupe; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.defaultDedupArgs; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.defaultFieldsArgs; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.defaultSortFieldArgs; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.defaultSortOptions; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.defaultStatsArgs; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.eval; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.exprList; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.field; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.filter; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.function; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.intLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.let; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.map; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.nullLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.projectWithArg; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.relation; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.rename; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.sort; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.sortOptions; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.stringLiteral; +import static java.util.Collections.emptyList; +import static org.junit.Assert.assertEquals; + +import com.amazon.opendistroforelasticsearch.sql.ast.Node; +import com.amazon.opendistroforelasticsearch.sql.ppl.antlr.PPLSyntaxParser; +import org.junit.Ignore; +import org.junit.Test; + +public class AstBuilderTest { + + private PPLSyntaxParser parser = new PPLSyntaxParser(); + private AstBuilder astBuilder = new AstBuilder(new AstExpressionBuilder()); + + @Test + public void testSearchCommand() { + assertEqual("search source=t a=1", + filter( + relation("t"), + compare("=", field("a"), intLiteral(1)) + ) + ); + } + + @Test + public void testSearchCommandString() { + assertEqual("search source=t a=\"a\"", + filter( + relation("t"), + compare("=", field("a"), stringLiteral("a")) + ) + ); + } + + @Test + public void testSearchCommandWithoutSearch() { + assertEqual("source=t a=1", + filter( + relation("t"), + compare("=", field("a"), intLiteral(1)) + ) + ); + } + + @Test + public void testSearchCommandWithFilterBeforeSource() { + assertEqual("search a=1 source=t", + filter( + relation("t"), + compare("=", field("a"), intLiteral(1)) + )); + } + + @Test + public void testWhereCommand() { + assertEqual("search source=t | where a=1", + filter( + relation("t"), + compare("=", field("a"), intLiteral(1)) + ) + ); + } + + @Test + public void testFieldsCommandWithoutArguments() { + assertEqual("source=t | fields f, g", + projectWithArg( + relation("t"), + defaultFieldsArgs(), + field("f"), field("g") + )); + } + + @Test + public void testFieldsCommandWithIncludeArguments() { + assertEqual("source=t | fields + f, g", + projectWithArg( + relation("t"), + defaultFieldsArgs(), + field("f"), field("g") + )); + } + + @Test + public void testFieldsCommandWithExcludeArguments() { + assertEqual("source=t | fields - f, g", + projectWithArg( + relation("t"), + exprList(argument("exclude", booleanLiteral(true))), + field("f"), field("g") + )); + } + + @Test + public void testRenameCommand() { + assertEqual("source=t | rename f as g", + rename( + relation("t"), + map("f", "g") + )); + } + + @Test + public void testRenameCommandWithMultiFields() { + assertEqual("source=t | rename f as g, h as i, j as k", + rename( + relation("t"), + map("f", "g"), + map("h", "i"), + map("j", "k") + )); + } + + @Test + public void testStatsCommand() { + assertEqual("source=t | stats count(a)", + agg( + relation("t"), + exprList( + aggregate("count", field("a")) + ), + emptyList(), + emptyList(), + defaultStatsArgs() + )); + } + + @Test + public void testStatsCommandWithByClause() { + assertEqual("source=t | stats count(a) by b DEDUP_SPLITVALUES=false", + agg( + relation("t"), + exprList( + aggregate("count", field("a")) + ), + emptyList(), + exprList(field("b")), + defaultStatsArgs() + )); + } + + @Test + public void testStatsCommandWithAlias() { + assertEqual("source=t | stats count(a) as alias", + rename( + agg( + relation("t"), + exprList( + aggregate("count", field("a")) + ), + emptyList(), + emptyList(), + defaultStatsArgs() + ), + map(aggregate("count", field("a")), field("alias")) + ) + ); + } + + @Test + public void testStatsCommandWithNestedFunctions() { + assertEqual("source=t | stats sum(a+b)", + agg( + relation("t"), + exprList( + aggregate( + "sum", + function("+", field("a"), field("b")) + )), + emptyList(), + emptyList(), + defaultStatsArgs() + )); + assertEqual("source=t | stats sum(abs(a)/2)", + agg( + relation("t"), + exprList( + aggregate( + "sum", + function( + "/", + function("abs", field("a")), + intLiteral(2) + ) + ) + ), + emptyList(), + emptyList(), + defaultStatsArgs() + )); + } + + @Test + public void testDedupCommand() { + assertEqual("source=t | dedup f1, f2", + dedupe( + relation("t"), + defaultDedupArgs(), + field("f1"), field("f2") + )); + } + + /** + * disable sortby from the dedup command syntax. + */ + @Ignore(value = "disable sortby from the dedup command syntax") + public void testDedupCommandWithSortby() { + assertEqual("source=t | dedup f1, f2 sortby f3", + agg( + relation("t"), + exprList(field("f1"), field("f2")), + exprList(field("f3", defaultSortFieldArgs())), + null, + defaultDedupArgs() + )); + } + + @Test + public void testSortCommand() { + assertEqual("source=t | sort f1, f2", + sort( + relation("t"), + defaultSortOptions(), + field("f1", defaultSortFieldArgs()), + field("f2", defaultSortFieldArgs()) + )); + } + + @Test + public void testSortCommandWithOptions() { + assertEqual("source=t | sort 100 - f1, + f2", + sort( + relation("t"), + sortOptions(100), + field("f1", exprList(argument("asc", booleanLiteral(false)), + argument("type", nullLiteral()))), + field("f2", defaultSortFieldArgs()) + )); + } + + @Test + public void testEvalCommand() { + assertEqual("source=t | eval r=abs(f)", + eval( + relation("t"), + let( + field("r"), + function("abs", field("f")) + ) + )); + } + + @Test + public void testIndexName() { + assertEqual("source=`log.2020.04.20.` a=1", + filter( + relation("log.2020.04.20."), + compare("=", field("a"), intLiteral(1)) + )); + } + + protected void assertEqual(String query, Node expectedPlan) { + Node actualPlan = plan(query); + assertEquals(expectedPlan, actualPlan); + } + + protected void assertEqual(String query, String expected) { + Node expectedPlan = plan(expected); + assertEqual(query, expectedPlan); + } + + private Node plan(String query) { + return astBuilder.visit(parser.analyzeSyntax(query)); + } +} diff --git a/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilderTest.java new file mode 100644 index 0000000000..86a061fe2f --- /dev/null +++ b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilderTest.java @@ -0,0 +1,411 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl.parser; + +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.agg; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.aggregate; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.and; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.argument; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.booleanLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.compare; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.defaultFieldsArgs; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.defaultSortFieldArgs; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.defaultSortOptions; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.defaultStatsArgs; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.doubleLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.equalTo; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.eval; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.exprList; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.field; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.filter; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.function; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.in; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.intLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.let; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.not; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.nullLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.or; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.projectWithArg; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.qualifiedName; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.relation; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.sort; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.stringLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.xor; +import static java.util.Collections.emptyList; + +import org.junit.Ignore; +import org.junit.Test; + +public class AstExpressionBuilderTest extends AstBuilderTest { + + @Test + public void testLogicalNotExpr() { + assertEqual("source=t not a=1", + filter( + relation("t"), + not( + compare("=", field("a"), intLiteral(1)) + ) + )); + } + + @Test + public void testLogicalOrExpr() { + assertEqual("source=t a=1 or b=2", + filter( + relation("t"), + or( + compare("=", field("a"), intLiteral(1)), + compare("=", field("b"), intLiteral(2)) + ) + )); + } + + @Test + public void testLogicalAndExpr() { + assertEqual("source=t a=1 and b=2", + filter( + relation("t"), + and( + compare("=", field("a"), intLiteral(1)), + compare("=", field("b"), intLiteral(2)) + ) + )); + } + + @Test + public void testLogicalAndExprWithoutKeywordAnd() { + assertEqual("source=t a=1 b=2", + filter( + relation("t"), + and( + compare("=", field("a"), intLiteral(1)), + compare("=", field("b"), intLiteral(2)) + ) + )); + } + + @Test + public void testLogicalXorExpr() { + assertEqual("source=t a=1 xor b=2", + filter( + relation("t"), + xor( + compare("=", field("a"), intLiteral(1)), + compare("=", field("b"), intLiteral(2)) + ) + )); + } + + @Test + public void testLogicalLikeExpr() { + assertEqual("source=t a like '_a%b%c_d_'", + filter( + relation("t"), + compare("like", field("a"), stringLiteral("_a%b%c_d_")) + )); + } + + /** + * Todo. search operator should not include functionCall, need to change antlr. + */ + @Ignore("search operator should not include functionCall, need to change antlr") + public void testEvalExpr() { + assertEqual("source=t f=abs(a)", + filter( + relation("t"), + equalTo( + field("f"), + function("abs", field("a")) + ) + )); + } + + @Test + public void testEvalFunctionExpr() { + assertEqual("source=t | eval f=abs(a)", + eval( + relation("t"), + let( + field("f"), + function("abs", field("a")) + ) + )); + } + + @Test + public void testEvalBinaryOperationExpr() { + assertEqual("source=t | eval f=a+b", + eval( + relation("t"), + let( + field("f"), + function("+", field("a"), field("b")) + ) + )); + assertEqual("source=t | eval f=(a+b)", + eval( + relation("t"), + let( + field("f"), + function("+", field("a"), field("b")) + ) + )); + } + + @Test + public void testLiteralValueBinaryOperationExpr() { + assertEqual("source=t | eval f=3+2", + eval( + relation("t"), + let( + field("f"), + function("+", intLiteral(3), intLiteral(2)) + ) + )); + } + + @Test + public void testCompareExpr() { + assertEqual("source=t a='b'", + filter( + relation("t"), + compare("=", field("a"), stringLiteral("b")) + )); + } + + @Test + public void testCompareFieldsExpr() { + assertEqual("source=t a>b", + filter( + relation("t"), + compare(">", field("a"), field("b")) + )); + } + + @Test + public void testInExpr() { + assertEqual("source=t f in (1, 2, 3)", + filter( + relation("t"), + in( + field("f"), + intLiteral(1), intLiteral(2), intLiteral(3)) + )); + } + + @Test + public void testFieldExpr() { + assertEqual("source=t | sort + f", + sort( + relation("t"), + defaultSortOptions(), + field("f", defaultSortFieldArgs()) + )); + } + + @Test + public void testSortFieldWithMinusKeyword() { + assertEqual("source=t | sort - f", + sort( + relation("t"), + defaultSortOptions(), + field( + "f", + argument("asc", booleanLiteral(false)), + argument("type", nullLiteral()) + ) + )); + } + + @Test + public void testSortFieldWithAutoKeyword() { + assertEqual("source=t | sort auto(f)", + sort( + relation("t"), + defaultSortOptions(), + field( + "f", + argument("asc", booleanLiteral(true)), + argument("type", stringLiteral("auto")) + ) + )); + } + + @Test + public void testSortFieldWithIpKeyword() { + assertEqual("source=t | sort ip(f)", + sort( + relation("t"), + defaultSortOptions(), + field( + "f", + argument("asc", booleanLiteral(true)), + argument("type", stringLiteral("ip")) + ) + )); + } + + @Test + public void testSortFieldWithNumKeyword() { + assertEqual("source=t | sort num(f)", + sort( + relation("t"), + defaultSortOptions(), + field( + "f", + argument("asc", booleanLiteral(true)), + argument("type", stringLiteral("num")) + ) + )); + } + + @Test + public void testSortFieldWithStrKeyword() { + assertEqual("source=t | sort str(f)", + sort( + relation("t"), + defaultSortOptions(), + field( + "f", + argument("asc", booleanLiteral(true)), + argument("type", stringLiteral("str")) + ) + )); + } + + @Test + public void testAggFuncCallExpr() { + assertEqual("source=t | stats avg(a) by b", + agg( + relation("t"), + exprList( + aggregate("avg", field("a")) + + ), + emptyList(), + exprList(field("b")), + defaultStatsArgs() + )); + } + + @Test + public void testPercentileAggFuncExpr() { + assertEqual("source=t | stats percentile<1>(a)", + agg( + relation("t"), + exprList( + aggregate( + "percentile", + field("a"), + argument("rank", intLiteral(1)) + ) + ), + emptyList(), + emptyList(), + defaultStatsArgs() + )); + } + + @Test + public void testEvalFuncCallExpr() { + assertEqual("source=t | eval f=abs(a)", + eval( + relation("t"), + let( + field("f"), + function("abs", field("a")) + ) + )); + } + + @Test + public void testNestedFieldName() { + assertEqual("source=t | fields field0.field1.field2", + projectWithArg( + relation("t"), + defaultFieldsArgs(), + field( + qualifiedName("field0", "field1", "field2") + ) + )); + } + + @Test + public void testFieldNameWithSpecialChars() { + assertEqual("source=t | fields `field-0`.`field#1`.`field*2`", + projectWithArg( + relation("t"), + defaultFieldsArgs(), + field( + qualifiedName("field-0", "field#1", "field*2") + ) + )); + } + + @Test + public void testStringLiteralExpr() { + assertEqual("source=t a=\"string\"", + filter( + relation("t"), + compare( + "=", + field("a"), + stringLiteral("string") + ) + )); + } + + @Test + public void testIntegerLiteralExpr() { + assertEqual("source=t a=1", + filter( + relation("t"), + compare( + "=", + field("a"), + intLiteral(1) + ) + )); + } + + @Test + public void testDoubleLiteralExpr() { + assertEqual("source=t b=0.1", + filter( + relation("t"), + compare( + "=", + field("b"), + doubleLiteral(0.1) + ) + )); + } + + @Test + public void testBooleanLiteralExpr() { + assertEqual("source=t a=true", + filter( + relation("t"), + compare( + "=", + field("a"), + booleanLiteral(true) + ) + )); + } + +} diff --git a/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/ArgumentFactoryTest.java b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/ArgumentFactoryTest.java new file mode 100644 index 0000000000..f2400b5ade --- /dev/null +++ b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/ArgumentFactoryTest.java @@ -0,0 +1,144 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl.utils; + +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.agg; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.aggregate; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.argument; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.booleanLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.dedupe; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.defaultSortFieldArgs; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.defaultSortOptions; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.exprList; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.field; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.intLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.projectWithArg; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.relation; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.sort; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.stringLiteral; +import static java.util.Collections.emptyList; + +import com.amazon.opendistroforelasticsearch.sql.ppl.parser.AstBuilderTest; +import org.junit.Test; + +public class ArgumentFactoryTest extends AstBuilderTest { + + @Test + public void testFieldsCommandArgument() { + assertEqual("source=t | fields - a", + projectWithArg( + relation("t"), + exprList(argument("exclude", booleanLiteral(true))), + field("a") + )); + } + + @Test + public void testFieldsCommandDefaultArgument() { + assertEqual("source=t | fields + a", "source=t | fields a"); + } + + @Test + public void testStatsCommandArgument() { + assertEqual( + "source=t | stats partitions=1 allnum=false delim=',' avg(a) dedup_splitvalues=true", + agg( + relation("t"), + exprList(aggregate("avg", field("a"))), + emptyList(), + emptyList(), + exprList( + argument("partitions", intLiteral(1)), + argument("allnum", booleanLiteral(false)), + argument("delim", stringLiteral(",")), + argument("dedupsplit", booleanLiteral(true)) + ) + )); + } + + @Test + public void testStatsCommandDefaultArgument() { + assertEqual( + "source=t | stats partitions=1 allnum=false delim=' ' avg(a) dedup_splitvalues=false", + "source=t | stats avg(a)"); + } + + @Test + public void testDedupCommandArgument() { + assertEqual("source=t | dedup 3 field0 keepempty=false consecutive=true", + dedupe( + relation("t"), + exprList( + argument("number", intLiteral(3)), + argument("keepempty", booleanLiteral(false)), + argument("consecutive", booleanLiteral(true)) + ), + field("field0") + )); + } + + @Test + public void testDedupCommandDefaultArgument() { + assertEqual( + "source=t | dedup 1 field0 keepempty=false consecutive=false", + "source=t | dedup field0" + ); + } + + @Test + public void testSortCommandArgument() { + assertEqual("source=t | sort 3 field0 desc", + sort( + relation("t"), + exprList( + argument("count", intLiteral(3)), + argument("desc", booleanLiteral(true)) + ), + field("field0", defaultSortFieldArgs()) + )); + assertEqual("source=t | sort 3 field0 d", "source=t | sort 3 field0 desc"); + } + + @Test + public void testSortCommandDefaultArgument() { + assertEqual( + "source=t | sort 1000 field0", + "source=t | sort field0" + ); + } + + @Test + public void testSortFieldArgument() { + assertEqual("source=t | sort - auto(field0)", + sort( + relation("t"), + defaultSortOptions(), + field( + "field0", + exprList( + argument("asc", booleanLiteral(false)), + argument("type", stringLiteral("auto")) + ) + ) + )); + } + + @Test + public void testNoArgConstructorForArgumentFactoryShouldPass() { + new ArgumentFactory(); + } + +} diff --git a/protocol/build.gradle b/protocol/build.gradle new file mode 100644 index 0000000000..8a90dea360 --- /dev/null +++ b/protocol/build.gradle @@ -0,0 +1,60 @@ +plugins { + id 'java' + id "io.freefair.lombok" + id 'jacoco' +} + +repositories { + mavenCentral() +} + +dependencies { + compile group: 'com.google.guava', name: 'guava', version: '23.0' + compile group: 'org.json', name: 'json', version: '20180813' //TODO: change to other JSON lib? + compile project(':core') + + testImplementation('org.junit.jupiter:junit-jupiter:5.6.2') + testCompile group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' + testCompile group: 'org.mockito', name: 'mockito-core', version: '3.3.3' + testCompile group: 'org.mockito', name: 'mockito-junit-jupiter', version: '3.3.3' +} + +test { + useJUnitPlatform() + testLogging { + events "passed", "skipped", "failed" + } +} + +jacoco { + toolVersion = "0.8.5" +} +jacocoTestReport { + reports { + html.enabled true + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it) + })) + } +} +test.finalizedBy(project.tasks.jacocoTestReport) + +jacocoTestCoverageVerification { + violationRules { + rule { + limit { + minimum = 1.0 + } + + } + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it) + })) + } +} +check.dependsOn jacocoTestCoverageVerification +jacocoTestCoverageVerification.dependsOn jacocoTestReport \ No newline at end of file diff --git a/protocol/lombok.config b/protocol/lombok.config new file mode 100644 index 0000000000..189c0bef98 --- /dev/null +++ b/protocol/lombok.config @@ -0,0 +1,3 @@ +# This file is generated by the 'io.freefair.lombok' Gradle plugin +config.stopBubbling = true +lombok.addLombokGeneratedAnnotation = true diff --git a/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResult.java b/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResult.java new file mode 100644 index 0000000000..45dfd5bf88 --- /dev/null +++ b/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResult.java @@ -0,0 +1,99 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.protocol.response; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import lombok.RequiredArgsConstructor; + +/** + * Query response that encapsulates query results and isolate {@link ExprValue} + * related from formatter implementation. + */ +@RequiredArgsConstructor +public class QueryResult implements Iterable { + + /** + * Results which are collection of expression. + */ + private final Collection exprValues; + + + /** + * size of results. + * @return size of results + */ + public int size() { + return exprValues.size(); + } + + /** + * Parse column name from results. + * + * @return mapping from column names to its expression type + */ + public Map columnNameTypes() { + if (exprValues.isEmpty()) { + return Collections.emptyMap(); + } + + // TODO: Need other way to extract header than inferring from data implicitly + Map tupleValue = getFirstTupleValue(); + return populateColumnNameAndTypes(tupleValue); + } + + @Override + public Iterator iterator() { + // Any chance to avoid copy for json response generation? + return exprValues.stream() + .map(ExprValueUtils::getTupleValue) + .map(Map::values) + .map(this::convertExprValuesToValues) + .iterator(); + } + + private Map getFirstTupleValue() { + // Assume expression is always tuple on first level + // and columns (keys) of all tuple values are exactly same + ExprValue firstValue = exprValues.iterator().next(); + return ExprValueUtils.getTupleValue(firstValue); + } + + private Map populateColumnNameAndTypes(Map tupleValue) { + // Use linked hashmap to maintain original order in tuple expression + Map colNameTypes = new LinkedHashMap<>(); + tupleValue.forEach((name, expr) -> colNameTypes.put(name, getTypeString(expr))); + return colNameTypes; + } + + private Object[] convertExprValuesToValues(Collection exprValues) { + return exprValues + .stream() + .map(ExprValue::value) + .toArray(Object[]::new); + } + + private String getTypeString(ExprValue exprValue) { + return exprValue.type().name().toLowerCase(); + } + +} diff --git a/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JsonResponseFormatter.java b/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JsonResponseFormatter.java new file mode 100644 index 0000000000..7b383dd0ca --- /dev/null +++ b/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/JsonResponseFormatter.java @@ -0,0 +1,78 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.protocol.response.format; + +import static com.amazon.opendistroforelasticsearch.sql.protocol.response.format.JsonResponseFormatter.Style.PRETTY; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.json.JSONObject; + +/** + * Abstract class for all JSON formatter. + * + * @param response generic type which could be DQL or DML response + */ +@RequiredArgsConstructor +public abstract class JsonResponseFormatter implements ResponseFormatter { + + /** + * JSON format styles: pretty format or compact format without indent and space. + */ + public enum Style { + PRETTY, COMPACT + } + + /** + * JSON format style. + */ + private final Style style; + + + @Override + public String format(R response) { + return jsonify(buildJsonObject(response)); + } + + @Override + public String format(Throwable t) { + JsonError error = new JsonError(t.getClass().getSimpleName(), + t.getMessage()); + return jsonify(error); + } + + /** + * Build JSON object to generate response json string. + * + * @param response response + * @return json object for response + */ + protected abstract Object buildJsonObject(R response); + + + private String jsonify(Object jsonObject) { + JSONObject json = new JSONObject(jsonObject); + return (style == PRETTY) ? json.toString(2) : json.toString(); + } + + @RequiredArgsConstructor + @Getter + public static class JsonError { + private final String type; + private final String reason; + } +} diff --git a/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/ResponseFormatter.java b/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/ResponseFormatter.java new file mode 100644 index 0000000000..4bee88f1c2 --- /dev/null +++ b/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/ResponseFormatter.java @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.protocol.response.format; + +/** + * Response formatter to format response to different formats. + */ +public interface ResponseFormatter { + + /** + * Format response into string in expected format. + * + * @param response response + * @return string with response content formatted + */ + String format(R response); + + /** + * Format an exception into string. + * + * @param t exception occurred + * @return string with exception content formatted + */ + String format(Throwable t); + +} diff --git a/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/SimpleJsonResponseFormatter.java b/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/SimpleJsonResponseFormatter.java new file mode 100644 index 0000000000..3a0bad1639 --- /dev/null +++ b/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/SimpleJsonResponseFormatter.java @@ -0,0 +1,96 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.protocol.response.format; + +import com.amazon.opendistroforelasticsearch.sql.protocol.response.QueryResult; +import java.util.List; +import lombok.Builder; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Singular; + +/** + * JSON response format with schema header and data rows. For example, + * + * 
+ *  {
+ *      "schema": [
+ *          {
+ *              "name": "name",
+ *              "type": "string"
+ *          }
+ *      ],
+ *      "datarows": [
+ *          ["John"],
+ *          ["Smith"]
+ *      ],
+ *      "total": 2,
+ *      "size": 2
+ *  }
+ *
+ */ +public class SimpleJsonResponseFormatter extends JsonResponseFormatter { + + public SimpleJsonResponseFormatter(Style style) { + super(style); + } + + @Override + public Object buildJsonObject(QueryResult response) { + JsonResponse.JsonResponseBuilder json = JsonResponse.builder(); + + json.total(response.size()) + .size(response.size()); + + response.columnNameTypes().forEach((name, type) -> json.column(new Column(name, type))); + + json.datarows(fetchDataRows(response)); + return json.build(); + } + + private Object[][] fetchDataRows(QueryResult response) { + Object[][] rows = new Object[response.size()][]; + int i = 0; + for (Object[] values : response) { + rows[i++] = values; + } + return rows; + } + + /** + * org.json requires these inner data classes be public (and static) + */ + @Builder + @Getter + public static class JsonResponse { + @Singular("column") + private final List schema; + + private final Object[][] datarows; + + private long total; + private long size; + } + + @RequiredArgsConstructor + @Getter + public static class Column { + private final String name; + private final String type; + } + +} diff --git a/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResultTest.java b/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResultTest.java new file mode 100644 index 0000000000..52cf905d9a --- /dev/null +++ b/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResultTest.java @@ -0,0 +1,95 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.protocol.response; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.tupleValue; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import java.util.Collections; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +class QueryResultTest { + + @Test + void size() { + QueryResult response = new QueryResult(Arrays.asList( + tupleValue(ImmutableMap.of("name", "John", "age", 20)), + tupleValue(ImmutableMap.of("name", "Allen", "age", 30)), + tupleValue(ImmutableMap.of("name", "Smith", "age", 40)) + )); + assertEquals(3, response.size()); + } + + @Test + void columnNameTypes() { + QueryResult response = new QueryResult(Collections.singletonList( + tupleValue(ImmutableMap.of("name", "John", "age", 20)) + )); + + assertEquals( + ImmutableMap.of("name", "string", "age", "integer"), + response.columnNameTypes() + ); + } + + @Test + void columnNameTypesFromEmptyExprValues() { + QueryResult response = new QueryResult(Collections.emptyList()); + assertTrue(response.columnNameTypes().isEmpty()); + } + + @Disabled("Need to figure out column headers in other way than inferring from data implicitly") + @Test + void columnNameTypesFromExprValuesWithMissing() { + QueryResult response = new QueryResult(Arrays.asList( + tupleValue(ImmutableMap.of("name", "John")), + tupleValue(ImmutableMap.of("name", "John", "age", 20)) + )); + + assertEquals( + ImmutableMap.of("name", "string", "age", "integer"), + response.columnNameTypes() + ); + } + + @Test + void iterate() { + QueryResult response = new QueryResult(Arrays.asList( + tupleValue(ImmutableMap.of("name", "John", "age", 20)), + tupleValue(ImmutableMap.of("name", "Allen", "age", 30)) + )); + + int i = 0; + for (Object[] objects : response) { + if (i == 0) { + assertArrayEquals(new Object[] {"John", 20}, objects); + } else if (i == 1) { + assertArrayEquals(new Object[] {"Allen", 30}, objects); + } else { + fail("More rows returned than expected"); + } + i++; + } + } + +} \ No newline at end of file diff --git a/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java b/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java new file mode 100644 index 0000000000..bd9dc9a817 --- /dev/null +++ b/protocol/src/test/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java @@ -0,0 +1,117 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.protocol.response.format; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.tupleValue; +import static com.amazon.opendistroforelasticsearch.sql.protocol.response.format.JsonResponseFormatter.Style.COMPACT; +import static com.amazon.opendistroforelasticsearch.sql.protocol.response.format.JsonResponseFormatter.Style.PRETTY; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.amazon.opendistroforelasticsearch.sql.protocol.response.QueryResult; +import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +class SimpleJsonResponseFormatterTest { + + @Test + void formatResponse() { + QueryResult response = + new QueryResult( + Arrays.asList( + tupleValue(ImmutableMap.of("firstname", "John", "age", 20)), + tupleValue(ImmutableMap.of("firstname", "Smith", "age", 30)))); + SimpleJsonResponseFormatter formatter = new SimpleJsonResponseFormatter(COMPACT); + assertEquals( + "{\"schema\":[{\"name\":\"firstname\",\"type\":\"string\"}," + + "{\"name\":\"age\",\"type\":\"integer\"}]," + + "\"total\":2,\"datarows\":[[\"John\",20],[\"Smith\",30]],\"size\":2}", + formatter.format(response)); + } + + @Test + void formatResponsePretty() { + QueryResult response = + new QueryResult( + Arrays.asList( + tupleValue(ImmutableMap.of("firstname", "John", "age", 20)), + tupleValue(ImmutableMap.of("firstname", "Smith", "age", 30)))); + SimpleJsonResponseFormatter formatter = new SimpleJsonResponseFormatter(PRETTY); + assertEquals( + "{\n" + + " \"schema\": [\n" + + " {\n" + + " \"name\": \"firstname\",\n" + + " \"type\": \"string\"\n" + + " },\n" + + " {\n" + + " \"name\": \"age\",\n" + + " \"type\": \"integer\"\n" + + " }\n" + + " ],\n" + + " \"total\": 2,\n" + + " \"datarows\": [\n" + + " [\n" + + " \"John\",\n" + + " 20\n" + + " ],\n" + + " [\n" + + " \"Smith\",\n" + + " 30\n" + + " ]\n" + + " ],\n" + + " \"size\": 2\n" + + "}", + formatter.format(response)); + } + + @Disabled("Need to figure out column headers in other way than inferring from data implicitly") + @Test + void formatResponseWithMissingValue() { + QueryResult response = + new QueryResult( + Arrays.asList( + tupleValue(ImmutableMap.of("firstname", "John")), + tupleValue(ImmutableMap.of("firstname", "Smith", "age", 30)))); + SimpleJsonResponseFormatter formatter = new SimpleJsonResponseFormatter(COMPACT); + assertEquals( + "{\"schema\":[{\"name\":\"firstname\",\"type\":\"string\"}," + + "{\"name\":\"age\",\"type\":\"integer\"}],\"total\":2," + + "\"datarows\":[{\"row\":[\"John\",null]},{\"row\":[\"Smith\",30]}],\"size\":2}", + formatter.format(response)); + } + + @Test + void formatError() { + SimpleJsonResponseFormatter formatter = new SimpleJsonResponseFormatter(COMPACT); + assertEquals( + "{\"reason\":\"This is an exception\",\"type\":\"RuntimeException\"}", + formatter.format(new RuntimeException("This is an exception"))); + } + + @Test + void formatErrorPretty() { + SimpleJsonResponseFormatter formatter = new SimpleJsonResponseFormatter(PRETTY); + assertEquals( + "{\n" + + " \"reason\": \"This is an exception\",\n" + + " \"type\": \"RuntimeException\"\n" + + "}", + formatter.format(new RuntimeException("This is an exception"))); + } +} diff --git a/release-notes/opendistro-elasticsearch-sql.release-notes-0.7.0.md b/release-notes/opendistro-elasticsearch-sql.release-notes-0.7.0.0.md similarity index 100% rename from release-notes/opendistro-elasticsearch-sql.release-notes-0.7.0.md rename to release-notes/opendistro-elasticsearch-sql.release-notes-0.7.0.0.md diff --git a/release-notes/opendistro-elasticsearch-sql.release-notes-0.8.0.md b/release-notes/opendistro-elasticsearch-sql.release-notes-0.8.0.0.md similarity index 100% rename from release-notes/opendistro-elasticsearch-sql.release-notes-0.8.0.md rename to release-notes/opendistro-elasticsearch-sql.release-notes-0.8.0.0.md diff --git a/release-notes/opendistro-elasticsearch-sql.release-notes-0.9.0.md b/release-notes/opendistro-elasticsearch-sql.release-notes-0.9.0.0.md similarity index 100% rename from release-notes/opendistro-elasticsearch-sql.release-notes-0.9.0.md rename to release-notes/opendistro-elasticsearch-sql.release-notes-0.9.0.0.md diff --git a/release-notes/opendistro-elasticsearch-sql.release-notes-1.0.0.md b/release-notes/opendistro-elasticsearch-sql.release-notes-1.0.0.0.md similarity index 100% rename from release-notes/opendistro-elasticsearch-sql.release-notes-1.0.0.md rename to release-notes/opendistro-elasticsearch-sql.release-notes-1.0.0.0.md diff --git a/release-notes/opendistro-elasticsearch-sql.release-notes-1.1.0.md b/release-notes/opendistro-elasticsearch-sql.release-notes-1.1.0.0.md similarity index 100% rename from release-notes/opendistro-elasticsearch-sql.release-notes-1.1.0.md rename to release-notes/opendistro-elasticsearch-sql.release-notes-1.1.0.0.md diff --git a/release-notes/opendistro-elasticsearch-sql.release-notes-1.2.0.md b/release-notes/opendistro-elasticsearch-sql.release-notes-1.2.0.0.md similarity index 100% rename from release-notes/opendistro-elasticsearch-sql.release-notes-1.2.0.md rename to release-notes/opendistro-elasticsearch-sql.release-notes-1.2.0.0.md diff --git a/release-notes/opendistro-elasticsearch-sql.release-notes-1.2.1.md b/release-notes/opendistro-elasticsearch-sql.release-notes-1.2.1.0.md similarity index 100% rename from release-notes/opendistro-elasticsearch-sql.release-notes-1.2.1.md rename to release-notes/opendistro-elasticsearch-sql.release-notes-1.2.1.0.md diff --git a/release-notes/opendistro-elasticsearch-sql.release-notes-1.3.0.md b/release-notes/opendistro-elasticsearch-sql.release-notes-1.3.0.0.md similarity index 100% rename from release-notes/opendistro-elasticsearch-sql.release-notes-1.3.0.md rename to release-notes/opendistro-elasticsearch-sql.release-notes-1.3.0.0.md diff --git a/release-notes/opendistro-elasticsearch-sql.release-notes-1.4.0.md b/release-notes/opendistro-elasticsearch-sql.release-notes-1.4.0.0.md similarity index 100% rename from release-notes/opendistro-elasticsearch-sql.release-notes-1.4.0.md rename to release-notes/opendistro-elasticsearch-sql.release-notes-1.4.0.0.md diff --git a/release-notes/opendistro-elasticsearch-sql.release-notes-1.6.0.md b/release-notes/opendistro-elasticsearch-sql.release-notes-1.6.0.0.md similarity index 100% rename from release-notes/opendistro-elasticsearch-sql.release-notes-1.6.0.md rename to release-notes/opendistro-elasticsearch-sql.release-notes-1.6.0.0.md diff --git a/release-notes/opendistro-elasticsearch-sql.release-notes-1.7.0.md b/release-notes/opendistro-elasticsearch-sql.release-notes-1.7.0.0.md similarity index 100% rename from release-notes/opendistro-elasticsearch-sql.release-notes-1.7.0.md rename to release-notes/opendistro-elasticsearch-sql.release-notes-1.7.0.0.md diff --git a/release-notes/opendistro-elasticsearch-sql.release-notes-1.8.0.md b/release-notes/opendistro-elasticsearch-sql.release-notes-1.8.0.0.md similarity index 100% rename from release-notes/opendistro-elasticsearch-sql.release-notes-1.8.0.md rename to release-notes/opendistro-elasticsearch-sql.release-notes-1.8.0.0.md diff --git a/release-notes/opendistro-elasticsearch-sql.release-notes-1.9.0.0.md b/release-notes/opendistro-elasticsearch-sql.release-notes-1.9.0.0.md new file mode 100644 index 0000000000..22322e845b --- /dev/null +++ b/release-notes/opendistro-elasticsearch-sql.release-notes-1.9.0.0.md @@ -0,0 +1,16 @@ +## 2020-06-23 Version 1.9.0.0 + +### Features +#### Elasticsearch Compatibility +* Feature [#532](https://github.com/opendistro-for-elasticsearch/sql/pull/532): Elasticsearch 7.8.0 compatibility (issue: [#531](https://github.com/opendistro-for-elasticsearch/sql/issues/531)) + +#### Documentation +* Feature [#486](https://github.com/opendistro-for-elasticsearch/sql/pull/486): Add Github badges to README + +### Enhancements +#### SQL Features +* Feature [#473](https://github.com/opendistro-for-elasticsearch/sql/pull/473): Support Integration Tests for Security enabled ODFE cluster + +### Bugfixes +* Bugfix [#489](https://github.com/opendistro-for-elasticsearch/sql/pull/489): Fix ANTLR grammar for negative integer and floating point number (issue: [#488](https://github.com/opendistro-for-elasticsearch/sql/issues/488)) +* Bugfix [#522](https://github.com/opendistro-for-elasticsearch/sql/pull/522): Bug fix, support long type for aggregation (issue: [#521](https://github.com/opendistro-for-elasticsearch/sql/issues/521)) diff --git a/settings.gradle b/settings.gradle index c6fdb47501..97aeab1de0 100644 --- a/settings.gradle +++ b/settings.gradle @@ -13,4 +13,16 @@ * permissions and limitations under the License. */ -rootProject.name = 'opendistro-sql' \ No newline at end of file +rootProject.name = 'opendistro-sql' + +include 'plugin' +include 'ppl' +include 'integ-test' +include 'common' +include 'elasticsearch' +include 'core' +include 'protocol' +include 'doctest' +include 'legacy' +include 'sql' + diff --git a/sql/build.gradle b/sql/build.gradle new file mode 100644 index 0000000000..56b18c010e --- /dev/null +++ b/sql/build.gradle @@ -0,0 +1,81 @@ +plugins { + id 'java' + id "io.freefair.lombok" + id 'jacoco' + id 'antlr' +} + +repositories { + mavenCentral() +} + +generateGrammarSource { + arguments += ['-visitor', '-package', 'com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser'] + source = sourceSets.main.antlr + outputDirectory = file("build/generated-src/antlr/main/com/amazon/opendistroforelasticsearch/sql/sql/antlr/parser") +} + +configurations { + compile { + extendsFrom = extendsFrom.findAll { it != configurations.antlr } + } +} + +dependencies { + antlr "org.antlr:antlr4:4.7.1" + + compile "org.antlr:antlr4-runtime:4.7.1" + compile group: 'com.google.guava', name: 'guava', version:'23.0' + compile group: 'org.json', name: 'json', version:'20180813' + compile group: 'org.springframework', name: 'spring-context', version: '5.2.5.RELEASE' + compile group: 'org.springframework', name: 'spring-beans', version: '5.2.5.RELEASE' + compile project(':common') + compile project(':core') + compile project(':protocol') + + testImplementation('org.junit.jupiter:junit-jupiter:5.6.2') + testCompile group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' + testCompile group: 'org.mockito', name: 'mockito-core', version: '3.3.3' + testCompile group: 'org.mockito', name: 'mockito-junit-jupiter', version: '3.3.3' +} + +test { + useJUnitPlatform() + testLogging { + events "passed", "skipped", "failed" + } +} + +jacoco { + toolVersion = "0.8.5" +} + +jacocoTestReport { + reports { + html.enabled true + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, + exclude: ['**/antlr/parser/**']) + })) + } +} +test.finalizedBy(project.tasks.jacocoTestReport) +jacocoTestCoverageVerification { + violationRules { + rule { + limit { + minimum = 1.0 + } + + } + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, + exclude: ['**/antlr/parser/**']) + })) + } +} +check.dependsOn jacocoTestCoverageVerification diff --git a/sql/lombok.config b/sql/lombok.config new file mode 100644 index 0000000000..189c0bef98 --- /dev/null +++ b/sql/lombok.config @@ -0,0 +1,3 @@ +# This file is generated by the 'io.freefair.lombok' Gradle plugin +config.stopBubbling = true +lombok.addLombokGeneratedAnnotation = true diff --git a/sql/src/main/antlr/OpenDistroSQLLexer.g4 b/sql/src/main/antlr/OpenDistroSQLLexer.g4 new file mode 100644 index 0000000000..61c13a3a30 --- /dev/null +++ b/sql/src/main/antlr/OpenDistroSQLLexer.g4 @@ -0,0 +1,345 @@ +/* +MySQL (Positive Technologies) grammar +The MIT License (MIT). +Copyright (c) 2015-2017, Ivan Kochurkin (kvanttt@gmail.com), Positive Technologies. +Copyright (c) 2017, Ivan Khudyashev (IHudyashov@ptsecurity.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +lexer grammar OpenDistroSQLLexer; + +channels { SQLCOMMENT, ERRORCHANNEL } + + +// SKIP + +SPACE: [ \t\r\n]+ -> channel(HIDDEN); +SPEC_SQL_COMMENT: '/*!' .+? '*/' -> channel(SQLCOMMENT); +COMMENT_INPUT: '/*' .*? '*/' -> channel(HIDDEN); +LINE_COMMENT: ( + ('-- ' | '#') ~[\r\n]* ('\r'? '\n' | EOF) + | '--' ('\r'? '\n' | EOF) + ) -> channel(HIDDEN); + + +// Keywords +// Common Keywords + +ALL: 'ALL'; +AND: 'AND'; +AS: 'AS'; +ASC: 'ASC'; +BETWEEN: 'BETWEEN'; +BY: 'BY'; +CASE: 'CASE'; +CAST: 'CAST'; +CROSS: 'CROSS'; +DATETIME: 'DATETIME'; +DELETE: 'DELETE'; +DESC: 'DESC'; +DESCRIBE: 'DESCRIBE'; +DISTINCT: 'DISTINCT'; +DOUBLE: 'DOUBLE'; +ELSE: 'ELSE'; +EXISTS: 'EXISTS'; +FALSE: 'FALSE'; +FLOAT: 'FLOAT'; +FROM: 'FROM'; +GROUP: 'GROUP'; +HAVING: 'HAVING'; +IN: 'IN'; +INNER: 'INNER'; +INT: 'INT'; +IS: 'IS'; +JOIN: 'JOIN'; +LEFT: 'LEFT'; +LIKE: 'LIKE'; +LIMIT: 'LIMIT'; +LONG: 'LONG'; +MATCH: 'MATCH'; +NATURAL: 'NATURAL'; +NOT: 'NOT'; +NULL_LITERAL: 'NULL'; +ON: 'ON'; +OR: 'OR'; +ORDER: 'ORDER'; +OUTER: 'OUTER'; +REGEXP: 'REGEXP'; +RIGHT: 'RIGHT'; +SELECT: 'SELECT'; +SHOW: 'SHOW'; +STRING: 'STRING'; +THEN: 'THEN'; +TRUE: 'TRUE'; +UNION: 'UNION'; +USING: 'USING'; +WHEN: 'WHEN'; +WHERE: 'WHERE'; + + +// OD SQL special keyword +MISSING: 'MISSING'; +EXCEPT: 'MINUS'; + + +// Group function Keywords + +AVG: 'AVG'; +COUNT: 'COUNT'; +MAX: 'MAX'; +MIN: 'MIN'; +SUM: 'SUM'; + + +// Common function Keywords + +SUBSTRING: 'SUBSTRING'; +TRIM: 'TRIM'; +YEAR: 'YEAR'; + + +// Keywords, but can be ID +// Common Keywords, but can be ID + +END: 'END'; +FULL: 'FULL'; +OFFSET: 'OFFSET'; + + +// PRIVILEGES + +TABLES: 'TABLES'; + + +// Common function names + +ABS: 'ABS'; +ACOS: 'ACOS'; +ADD: 'ADD'; +ASCII: 'ASCII'; +ASIN: 'ASIN'; +ATAN: 'ATAN'; +ATAN2: 'ATAN2'; +CBRT: 'CBRT'; +CEIL: 'CEIL'; +CONCAT: 'CONCAT'; +CONCAT_WS: 'CONCAT_WS'; +COS: 'COS'; +COSH: 'COSH'; +COT: 'COT'; +CURDATE: 'CURDATE'; +DATE: 'DATE'; +DATE_FORMAT: 'DATE_FORMAT'; +DAYOFMONTH: 'DAYOFMONTH'; +DEGREES: 'DEGREES'; +E: 'E'; +EXP: 'EXP'; +EXPM1: 'EXPM1'; +FLOOR: 'FLOOR'; +IF: 'IF'; +IFNULL: 'IFNULL'; +ISNULL: 'ISNULL'; +LENGTH: 'LENGTH'; +LN: 'LN'; +LOCATE: 'LOCATE'; +LOG: 'LOG'; +LOG10: 'LOG10'; +LOG2: 'LOG2'; +LOWER: 'LOWER'; +LTRIM: 'LTRIM'; +MAKETIME: 'MAKETIME'; +MODULUS: 'MODULUS'; +MONTH: 'MONTH'; +MONTHNAME: 'MONTHNAME'; +MULTIPLY: 'MULTIPLY'; +NOW: 'NOW'; +PI: 'PI'; +POW: 'POW'; +POWER: 'POWER'; +RADIANS: 'RADIANS'; +RAND: 'RAND'; +REPLACE: 'REPLACE'; +RINT: 'RINT'; +ROUND: 'ROUND'; +RTRIM: 'RTRIM'; +SIGN: 'SIGN'; +SIGNUM: 'SIGNUM'; +SIN: 'SIN'; +SINH: 'SINH'; +SQRT: 'SQRT'; +SUBTRACT: 'SUBTRACT'; +TAN: 'TAN'; +TIMESTAMP: 'TIMESTAMP'; +UPPER: 'UPPER'; + +D: 'D'; +T: 'T'; +TS: 'TS'; +LEFT_BRACE: '{'; +RIGHT_BRACE: '}'; + + +// OD SQL special functions +DATE_HISTOGRAM: 'DATE_HISTOGRAM'; +DAY_OF_MONTH: 'DAY_OF_MONTH'; +DAY_OF_YEAR: 'DAY_OF_YEAR'; +DAY_OF_WEEK: 'DAY_OF_WEEK'; +EXCLUDE: 'EXCLUDE'; +EXTENDED_STATS: 'EXTENDED_STATS'; +FIELD: 'FIELD'; +FILTER: 'FILTER'; +GEO_BOUNDING_BOX: 'GEO_BOUNDING_BOX'; +GEO_CELL: 'GEO_CELL'; +GEO_DISTANCE: 'GEO_DISTANCE'; +GEO_DISTANCE_RANGE: 'GEO_DISTANCE_RANGE'; +GEO_INTERSECTS: 'GEO_INTERSECTS'; +GEO_POLYGON: 'GEO_POLYGON'; +HISTOGRAM: 'HISTOGRAM'; +HOUR_OF_DAY: 'HOUR_OF_DAY'; +INCLUDE: 'INCLUDE'; +IN_TERMS: 'IN_TERMS'; +MATCHPHRASE: 'MATCHPHRASE'; +MATCH_PHRASE: 'MATCH_PHRASE'; +MATCHQUERY: 'MATCHQUERY'; +MATCH_QUERY: 'MATCH_QUERY'; +MINUTE_OF_DAY: 'MINUTE_OF_DAY'; +MINUTE_OF_HOUR: 'MINUTE_OF_HOUR'; +MONTH_OF_YEAR: 'MONTH_OF_YEAR'; +MULTIMATCH: 'MULTIMATCH'; +MULTI_MATCH: 'MULTI_MATCH'; +NESTED: 'NESTED'; +PERCENTILES: 'PERCENTILES'; +REGEXP_QUERY: 'REGEXP_QUERY'; +REVERSE_NESTED: 'REVERSE_NESTED'; +QUERY: 'QUERY'; +RANGE: 'RANGE'; +SCORE: 'SCORE'; +SECOND_OF_MINUTE: 'SECOND_OF_MINUTE'; +STATS: 'STATS'; +TERM: 'TERM'; +TERMS: 'TERMS'; +TOPHITS: 'TOPHITS'; +WEEK_OF_YEAR: 'WEEK_OF_YEAR'; +WILDCARDQUERY: 'WILDCARDQUERY'; +WILDCARD_QUERY: 'WILDCARD_QUERY'; + + +// Operators + +// Operators. Arithmetics + +STAR: '*'; +DIVIDE: '/'; +MODULE: '%'; +PLUS: '+'; +MINUS: '-'; +DIV: 'DIV'; +MOD: 'MOD'; + + +// Operators. Comparation + +EQUAL_SYMBOL: '='; +GREATER_SYMBOL: '>'; +LESS_SYMBOL: '<'; +EXCLAMATION_SYMBOL: '!'; + + +// Operators. Bit + +BIT_NOT_OP: '~'; +BIT_OR_OP: '|'; +BIT_AND_OP: '&'; +BIT_XOR_OP: '^'; + + +// Constructors symbols + +DOT: '.'; +LR_BRACKET: '('; +RR_BRACKET: ')'; +COMMA: ','; +SEMI: ';'; +AT_SIGN: '@'; +ZERO_DECIMAL: '0'; +ONE_DECIMAL: '1'; +TWO_DECIMAL: '2'; +SINGLE_QUOTE_SYMB: '\''; +DOUBLE_QUOTE_SYMB: '"'; +REVERSE_QUOTE_SYMB: '`'; +COLON_SYMB: ':'; + + +// Literal Primitives + +START_NATIONAL_STRING_LITERAL: 'N' SQUOTA_STRING; +STRING_LITERAL: DQUOTA_STRING | SQUOTA_STRING | BQUOTA_STRING; +DECIMAL_LITERAL: DEC_DIGIT+; +HEXADECIMAL_LITERAL: 'X' '\'' (HEX_DIGIT HEX_DIGIT)+ '\'' + | '0X' HEX_DIGIT+; + +REAL_LITERAL: (DEC_DIGIT+)? '.' DEC_DIGIT+ + | DEC_DIGIT+ '.' EXPONENT_NUM_PART + | (DEC_DIGIT+)? '.' (DEC_DIGIT+ EXPONENT_NUM_PART) + | DEC_DIGIT+ EXPONENT_NUM_PART; +NULL_SPEC_LITERAL: '\\' 'N'; +BIT_STRING: BIT_STRING_L; + + + +// Hack for dotID +// Prevent recognize string: .123somelatin AS ((.123), FLOAT_LITERAL), ((somelatin), ID) +// it must recoginze: .123somelatin AS ((.), DOT), (123somelatin, ID) + +DOT_ID: '.' ID_LITERAL; + + + +// Identifiers + +ID: ID_LITERAL; +// DOUBLE_QUOTE_ID: '"' ~'"'+ '"'; +REVERSE_QUOTE_ID: '`' ~'`'+ '`'; +STRING_USER_NAME: ( + SQUOTA_STRING | DQUOTA_STRING + | BQUOTA_STRING | ID_LITERAL + ) '@' + ( + SQUOTA_STRING | DQUOTA_STRING + | BQUOTA_STRING | ID_LITERAL + ); + + +// Fragments for Literal primitives + +fragment EXPONENT_NUM_PART: 'E' [-+]? DEC_DIGIT+; +fragment ID_LITERAL: [A-Z_$0-9@]*?[A-Z_$]+?[A-Z_$\-0-9]*; +fragment DQUOTA_STRING: '"' ( '\\'. | '""' | ~('"'| '\\') )* '"'; +fragment SQUOTA_STRING: '\'' ('\\'. | '\'\'' | ~('\'' | '\\'))* '\''; +fragment BQUOTA_STRING: '`' ( '\\'. | '``' | ~('`'|'\\'))* '`'; +fragment HEX_DIGIT: [0-9A-F]; +fragment DEC_DIGIT: [0-9]; +fragment BIT_STRING_L: 'B' '\'' [01]+ '\''; + + + +// Last tokens must generate Errors + +ERROR_RECOGNITION: . -> channel(ERRORCHANNEL); diff --git a/sql/src/main/antlr/OpenDistroSQLParser.g4 b/sql/src/main/antlr/OpenDistroSQLParser.g4 new file mode 100644 index 0000000000..fb2d7eb83e --- /dev/null +++ b/sql/src/main/antlr/OpenDistroSQLParser.g4 @@ -0,0 +1,154 @@ +/* +MySQL (Positive Technologies) grammar +The MIT License (MIT). +Copyright (c) 2015-2017, Ivan Kochurkin (kvanttt@gmail.com), Positive Technologies. +Copyright (c) 2017, Ivan Khudyashev (IHudyashov@ptsecurity.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +parser grammar OpenDistroSQLParser; + +options { tokenVocab=OpenDistroSQLLexer; } + + +// Top Level Description + +// Root rule +root + : sqlStatement? EOF + ; + +// Only SELECT +sqlStatement + : dmlStatement + ; + +dmlStatement + : selectStatement + ; + + +// Data Manipulation Language + +// Primary DML Statements + +selectStatement + : querySpecification #simpleSelect + ; + + +// Select Statement's Details + +querySpecification + : SELECT selectElements + ; + +selectElements + : selectElement (COMMA selectElement)* + ; + +selectElement + : expression #selectExpressionElement + ; + + +// Literals + +constant + : stringLiteral #string + | sign? decimalLiteral #signedDecimal + | sign? realLiteral #signedReal + | booleanLiteral #boolean + // Doesn't support the following types for now + //| nullLiteral #null + //| BIT_STRING + //| NOT? nullLiteral=(NULL_LITERAL | NULL_SPEC_LITERAL) + //| LEFT_BRACE dateType=(D | T | TS | DATE | TIME | TIMESTAMP) stringLiteral RIGHT_BRACE + ; + +decimalLiteral + : DECIMAL_LITERAL | ZERO_DECIMAL | ONE_DECIMAL | TWO_DECIMAL + ; + +stringLiteral + : ( + STRING_LITERAL + | START_NATIONAL_STRING_LITERAL + ) STRING_LITERAL+ + | ( + STRING_LITERAL + | START_NATIONAL_STRING_LITERAL + ) + ; + +booleanLiteral + : TRUE | FALSE + ; + +realLiteral + : REAL_LITERAL + ; + +sign + : PLUS | MINUS + ; + +nullLiteral + : NULL_LITERAL + ; + +// Expressions, predicates + +// Simplified approach for expression +expression + : predicate #predicateExpression + ; + +predicate + : expressionAtom #expressionAtomPredicate + ; + +expressionAtom + : constant #constantExpressionAtom + | functionCall #functionCallExpressionAtom + | LR_BRACKET expression RR_BRACKET #nestedExpressionAtom + | left=expressionAtom mathOperator right=expressionAtom #mathExpressionAtom + ; + +mathOperator + : PLUS | MINUS | STAR | DIVIDE | MODULE + ; + +functionCall + : scalarFunctionName LR_BRACKET functionArgs? RR_BRACKET #scalarFunctionCall + ; + +scalarFunctionName + : ABS + ; + +functionArgs + : functionArg (COMMA functionArg)* + ; + +functionArg + : expression + ; + diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java new file mode 100644 index 0000000000..599d02bddb --- /dev/null +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java @@ -0,0 +1,115 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql; + +import com.amazon.opendistroforelasticsearch.sql.analysis.AnalysisContext; +import com.amazon.opendistroforelasticsearch.sql.analysis.Analyzer; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; +import com.amazon.opendistroforelasticsearch.sql.common.response.ResponseListener; +import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine; +import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.QueryResponse; +import com.amazon.opendistroforelasticsearch.sql.planner.Planner; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.SQLSyntaxParser; +import com.amazon.opendistroforelasticsearch.sql.sql.domain.SQLQueryRequest; +import com.amazon.opendistroforelasticsearch.sql.sql.parser.AstBuilder; +import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; +import org.antlr.v4.runtime.tree.ParseTree; + +/** + * SQL service. + */ +public class SQLService { + + private final SQLSyntaxParser parser; + + private final Analyzer analyzer; + + private final StorageEngine storageEngine; + + private final ExecutionEngine executionEngine; + + /** + * Initialize SQL service. + * @param parser SQL syntax parser + * @param analyzer AST analyzer + * @param storageEngine storage engine + * @param executionEngine execution engine + */ + public SQLService(SQLSyntaxParser parser, Analyzer analyzer, + StorageEngine storageEngine, ExecutionEngine executionEngine) { + this.parser = parser; + this.analyzer = analyzer; + this.storageEngine = storageEngine; + this.executionEngine = executionEngine; + } + + /** + * Parse, analyze, plan and execute the query. + * @param request SQL query request + * @param listener callback listener + */ + public void execute(SQLQueryRequest request, ResponseListener listener) { + try { + executionEngine.execute( + plan( + analyze( + parse(request.getQuery()))), listener); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Given AST, run the remaining steps to execute it. + * @param ast AST + * @param listener callback listener + */ + public void execute(UnresolvedPlan ast, ResponseListener listener) { + try { + executionEngine.execute( + plan( + analyze(ast)), listener); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Parse query and convert parse tree (CST) to abstract syntax tree (AST). + */ + public UnresolvedPlan parse(String query) { + ParseTree cst = parser.parse(query); + return cst.accept(new AstBuilder()); + } + + /** + * Analyze abstract syntax to generate logical plan. + */ + public LogicalPlan analyze(UnresolvedPlan ast) { + return analyzer.analyze(ast, new AnalysisContext()); + } + + /** + * Generate optimal physical plan from logical plan. + */ + public PhysicalPlan plan(LogicalPlan logicalPlan) { + return new Planner(storageEngine).plan(logicalPlan); + } + +} diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParser.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParser.java new file mode 100644 index 0000000000..1a6ad372e1 --- /dev/null +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParser.java @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql.antlr; + +import com.amazon.opendistroforelasticsearch.sql.common.antlr.CaseInsensitiveCharStream; +import com.amazon.opendistroforelasticsearch.sql.common.antlr.SyntaxAnalysisErrorListener; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLLexer; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.tree.ParseTree; + +/** + * SQL syntax parser which encapsulates an ANTLR parser. + */ +public class SQLSyntaxParser { + + /** + * Parse a SQL query by ANTLR parser. + * @param query a SQL query + * @return parse tree root + */ + public ParseTree parse(String query) { + OpenDistroSQLLexer lexer = new OpenDistroSQLLexer(new CaseInsensitiveCharStream(query)); + OpenDistroSQLParser parser = new OpenDistroSQLParser(new CommonTokenStream(lexer)); + parser.addErrorListener(new SyntaxAnalysisErrorListener()); + return parser.root(); + } + +} diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/config/SQLServiceConfig.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/config/SQLServiceConfig.java new file mode 100644 index 0000000000..e7584eccc9 --- /dev/null +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/config/SQLServiceConfig.java @@ -0,0 +1,59 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql.config; + +import com.amazon.opendistroforelasticsearch.sql.analysis.Analyzer; +import com.amazon.opendistroforelasticsearch.sql.analysis.ExpressionAnalyzer; +import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine; +import com.amazon.opendistroforelasticsearch.sql.expression.config.ExpressionConfig; +import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository; +import com.amazon.opendistroforelasticsearch.sql.sql.SQLService; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.SQLSyntaxParser; +import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +/** + * SQL service configuration for Spring container initialization. + */ +@Configuration +@Import({ExpressionConfig.class}) +public class SQLServiceConfig { + + @Autowired + private StorageEngine storageEngine; + + @Autowired + private ExecutionEngine executionEngine; + + @Autowired + private BuiltinFunctionRepository functionRepository; + + @Bean + public Analyzer analyzer() { + return new Analyzer(new ExpressionAnalyzer(functionRepository), storageEngine); + } + + @Bean + public SQLService sqlService() { + return new SQLService(new SQLSyntaxParser(), analyzer(), storageEngine, executionEngine); + } + +} + diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequest.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequest.java new file mode 100644 index 0000000000..5857a0933f --- /dev/null +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequest.java @@ -0,0 +1,86 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql.domain; + +import com.google.common.base.Strings; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; +import org.json.JSONObject; + +/** + * SQL query request. + */ +@ToString +@Getter +@EqualsAndHashCode +@RequiredArgsConstructor +public class SQLQueryRequest { + + /** + * JSON payload in REST request. + */ + private final JSONObject jsonContent; + + /** + * SQL query. + */ + private final String query; + + /** + * Request path. + */ + private final String path; + + /** + * Request format. + */ + private final String format; + + /** + * Pre-check if the request can be supported by meeting the following criteria: + * 1.Not explain request + * 2.Only "query" field in payload. In other word, it's not a cursor request + * (with either "fetch_size" or "cursor" field) or request with extra field + * such as "filter". + * 3.Response format expected is default JDBC format. + * + * @return true if supported. + */ + public boolean isSupported() { + return !isExplainRequest() + && isOnlyQueryFieldInPayload() + && isDefaultFormat(); + } + + private boolean isExplainRequest() { + return path.endsWith("/_explain"); + } + + private boolean isOnlyQueryFieldInPayload() { + return (jsonContent.keySet().size() == 1 && jsonContent.has("query")) + || (jsonContent.keySet().size() == 2 && jsonContent.has("query") + && jsonContent.has("fetch_size") && jsonContent.getInt("fetch_size") == 0); + } + + private boolean isDefaultFormat() { + return Strings.isNullOrEmpty(format) || "jdbc".equalsIgnoreCase(format); + } + +} diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilder.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilder.java new file mode 100644 index 0000000000..40f42293d2 --- /dev/null +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilder.java @@ -0,0 +1,64 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql.parser; + +import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.SimpleSelectContext; + +import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Project; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Values; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParserBaseVisitor; +import com.google.common.collect.ImmutableList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import org.antlr.v4.runtime.tree.ParseTree; + +/** + * Abstract syntax tree (AST) builder. + */ +public class AstBuilder extends OpenDistroSQLParserBaseVisitor { + + private final AstExpressionBuilder expressionBuilder = new AstExpressionBuilder(); + + @Override + public UnresolvedPlan visitSimpleSelect(SimpleSelectContext ctx) { + List selectElements = ctx.querySpecification().selectElements().children; + Project project = new Project(selectElements.stream() + .map(this::visitAstExpression) + .filter(Objects::nonNull) + .collect(Collectors.toList())); + + // Attach an Values operator with only a empty row inside so that + // Project operator can have a chance to evaluate its expression + // though the evaluation doesn't have any dependency on what's in Values. + Values emptyValue = new Values(ImmutableList.of(Collections.emptyList())); + return project.attach(emptyValue); + } + + @Override + protected UnresolvedPlan aggregateResult(UnresolvedPlan aggregate, UnresolvedPlan nextResult) { + return nextResult != null ? nextResult : aggregate; + } + + private UnresolvedExpression visitAstExpression(ParseTree tree) { + return expressionBuilder.visit(tree); + } + +} diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java new file mode 100644 index 0000000000..057d2dbf2f --- /dev/null +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java @@ -0,0 +1,85 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql.parser; + +import static com.amazon.opendistroforelasticsearch.sql.common.utils.StringUtils.unquoteIdentifier; +import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.BooleanContext; +import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.MathExpressionAtomContext; +import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.ScalarFunctionCallContext; +import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.SignedDecimalContext; +import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.SignedRealContext; +import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.StringContext; + +import com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Function; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.NestedExpressionAtomContext; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParserBaseVisitor; +import java.util.Arrays; +import java.util.stream.Collectors; + +/** + * Expression builder to parse text to expression in AST. + */ +public class AstExpressionBuilder extends OpenDistroSQLParserBaseVisitor { + + @Override + public UnresolvedExpression visitMathExpressionAtom(MathExpressionAtomContext ctx) { + return new Function( + ctx.mathOperator().getText(), + Arrays.asList(visit(ctx.left), visit(ctx.right)) + ); + } + + @Override + public UnresolvedExpression visitNestedExpressionAtom(NestedExpressionAtomContext ctx) { + return visit(ctx.expression()); // Discard parenthesis around + } + + @Override + public UnresolvedExpression visitScalarFunctionCall(ScalarFunctionCallContext ctx) { + return new Function( + ctx.scalarFunctionName().getText(), + ctx.functionArgs() + .functionArg() + .stream() + .map(this::visitFunctionArg) + .collect(Collectors.toList()) + ); + } + + @Override + public UnresolvedExpression visitString(StringContext ctx) { + return AstDSL.stringLiteral(unquoteIdentifier(ctx.getText())); + } + + @Override + public UnresolvedExpression visitSignedDecimal(SignedDecimalContext ctx) { + return AstDSL.intLiteral(Integer.valueOf(ctx.getText())); + } + + @Override + public UnresolvedExpression visitSignedReal(SignedRealContext ctx) { + return AstDSL.doubleLiteral(Double.valueOf(ctx.getText())); + } + + @Override + public UnresolvedExpression visitBoolean(BooleanContext ctx) { + return AstDSL.booleanLiteral(Boolean.valueOf(ctx.getText())); + } + +} diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLServiceTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLServiceTest.java new file mode 100644 index 0000000000..ce563414ff --- /dev/null +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLServiceTest.java @@ -0,0 +1,152 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql; + +import static com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine.QueryResponse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; + +import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; +import com.amazon.opendistroforelasticsearch.sql.common.response.ResponseListener; +import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.SQLSyntaxParser; +import com.amazon.opendistroforelasticsearch.sql.sql.config.SQLServiceConfig; +import com.amazon.opendistroforelasticsearch.sql.sql.domain.SQLQueryRequest; +import com.amazon.opendistroforelasticsearch.sql.sql.parser.AstBuilder; +import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; +import java.util.Collections; +import org.antlr.v4.runtime.tree.ParseTree; +import org.json.JSONObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; + +@ExtendWith(MockitoExtension.class) +class SQLServiceTest { + + private AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); + + private SQLService sqlService; + + @Mock + private StorageEngine storageEngine; + + @Mock + private ExecutionEngine executionEngine; + + @BeforeEach + public void setUp() { + context.registerBean(StorageEngine.class, () -> storageEngine); + context.registerBean(ExecutionEngine.class, () -> executionEngine); + context.register(SQLServiceConfig.class); + context.refresh(); + sqlService = context.getBean(SQLService.class); + } + + @Test + public void canExecuteSqlQuery() { + doAnswer(invocation -> { + ResponseListener listener = invocation.getArgument(1); + listener.onResponse(new QueryResponse(Collections.emptyList())); + return null; + }).when(executionEngine).execute(any(), any()); + + sqlService.execute( + new SQLQueryRequest(new JSONObject(), "SELECT 123", "_opendistro/_sql", "jdbc"), + new ResponseListener() { + @Override + public void onResponse(QueryResponse response) { + assertNotNull(response); + } + + @Override + public void onFailure(Exception e) { + fail(e); + } + }); + } + + @Test + public void canExecuteFromAst() { + doAnswer(invocation -> { + ResponseListener listener = invocation.getArgument(1); + listener.onResponse(new QueryResponse(Collections.emptyList())); + return null; + }).when(executionEngine).execute(any(), any()); + + ParseTree parseTree = new SQLSyntaxParser().parse("SELECT 123"); + UnresolvedPlan ast = parseTree.accept(new AstBuilder()); + + sqlService.execute(ast, + new ResponseListener() { + @Override + public void onResponse(QueryResponse response) { + assertNotNull(response); + } + + @Override + public void onFailure(Exception e) { + fail(e); + } + }); + } + + @Test + public void canCaptureErrorDuringExecution() { + sqlService.execute( + new SQLQueryRequest(new JSONObject(), "SELECT", "_opendistro/_sql", ""), + new ResponseListener() { + @Override + public void onResponse(QueryResponse response) { + fail(); + } + + @Override + public void onFailure(Exception e) { + assertNotNull(e); + } + }); + } + + @Test + public void canCaptureErrorDuringExecutionFromAst() { + doThrow(new RuntimeException()).when(executionEngine).execute(any(), any()); + + ParseTree parseTree = new SQLSyntaxParser().parse("SELECT 123"); + UnresolvedPlan ast = parseTree.accept(new AstBuilder()); + + sqlService.execute(ast, + new ResponseListener() { + @Override + public void onResponse(QueryResponse response) { + fail(); + } + + @Override + public void onFailure(Exception e) { + assertNotNull(e); + } + }); + } + +} \ No newline at end of file diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java new file mode 100644 index 0000000000..7bd8d93b2b --- /dev/null +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java @@ -0,0 +1,41 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql.antlr; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.amazon.opendistroforelasticsearch.sql.common.antlr.SyntaxCheckException; +import org.antlr.v4.runtime.tree.ParseTree; +import org.junit.jupiter.api.Test; + +class SQLSyntaxParserTest { + + private final SQLSyntaxParser parser = new SQLSyntaxParser(); + + @Test + public void canParseSelectLiterals() { + ParseTree parseTree = parser.parse("SELECT 123, 'hello'"); + assertNotNull(parseTree); + } + + @Test + public void canNotParseInvalidSelect() { + assertThrows(SyntaxCheckException.class, () -> parser.parse("SELECT * FROM test")); + } + +} \ No newline at end of file diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/config/SQLServiceConfigTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/config/SQLServiceConfigTest.java new file mode 100644 index 0000000000..b254cbe312 --- /dev/null +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/config/SQLServiceConfigTest.java @@ -0,0 +1,31 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql.config; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import org.junit.jupiter.api.Test; + +class SQLServiceConfigTest { + + @Test + public void shouldReturnSQLService() { + SQLServiceConfig config = new SQLServiceConfig(); + assertNotNull(config.sqlService()); + } + +} \ No newline at end of file diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequestTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequestTest.java new file mode 100644 index 0000000000..6838b6be73 --- /dev/null +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/domain/SQLQueryRequestTest.java @@ -0,0 +1,121 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql.domain; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.json.JSONObject; +import org.junit.jupiter.api.Test; + +public class SQLQueryRequestTest { + + @Test + public void shouldSupportQuery() { + SQLQueryRequest request = SQLQueryRequestBuilder.request("SELECT 1").build(); + assertTrue(request.isSupported()); + } + + @Test + public void shouldSupportQueryWithJDBCFormat() { + SQLQueryRequest request = SQLQueryRequestBuilder.request("SELECT 1") + .format("jdbc") + .build(); + assertTrue(request.isSupported()); + } + + @Test + public void shouldSupportQueryWithZeroFetchSize() { + SQLQueryRequest request = + SQLQueryRequestBuilder.request("SELECT 1") + .jsonContent("{\"query\": \"SELECT 1\", \"fetch_size\": 0}") + .build(); + assertTrue(request.isSupported()); + } + + @Test + public void shouldNotSupportExplain() { + SQLQueryRequest explainRequest = + SQLQueryRequestBuilder.request("SELECT 1") + .path("_opendistro/_sql/_explain") + .build(); + assertFalse(explainRequest.isSupported()); + } + + @Test + public void shouldNotSupportCursorRequest() { + SQLQueryRequest fetchSizeRequest = + SQLQueryRequestBuilder.request("SELECT 1") + .jsonContent("{\"query\": \"SELECT 1\", \"fetch_size\": 5}") + .build(); + assertFalse(fetchSizeRequest.isSupported()); + + SQLQueryRequest cursorRequest = + SQLQueryRequestBuilder.request("SELECT 1") + .jsonContent("{\"cursor\": \"abcdefgh...\"}") + .build(); + assertFalse(cursorRequest.isSupported()); + } + + @Test + public void shouldNotSupportCSVFormat() { + SQLQueryRequest csvRequest = + SQLQueryRequestBuilder.request("SELECT 1") + .format("csv") + .build(); + assertFalse(csvRequest.isSupported()); + } + + /** + * SQL query request build helper to improve test data setup readability. + */ + private static class SQLQueryRequestBuilder { + private String jsonContent; + private String query; + private String path = "_/opendistro/_sql"; + private String format; + + static SQLQueryRequestBuilder request(String query) { + SQLQueryRequestBuilder builder = new SQLQueryRequestBuilder(); + builder.query = query; + return builder; + } + + SQLQueryRequestBuilder jsonContent(String jsonContent) { + this.jsonContent = jsonContent; + return this; + } + + SQLQueryRequestBuilder path(String path) { + this.path = path; + return this; + } + + SQLQueryRequestBuilder format(String format) { + this.format = format; + return this; + } + + SQLQueryRequest build() { + if (jsonContent == null) { + jsonContent = "{\"query\": \"" + query + "\"}"; + } + return new SQLQueryRequest(new JSONObject(jsonContent), query, path, format); + } + } + +} \ No newline at end of file diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilderTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilderTest.java new file mode 100644 index 0000000000..a5546db97f --- /dev/null +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilderTest.java @@ -0,0 +1,64 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql.parser; + +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.booleanLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.doubleLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.intLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.project; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.stringLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.values; +import static java.util.Collections.emptyList; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.SQLSyntaxParser; +import org.antlr.v4.runtime.tree.ParseTree; +import org.junit.jupiter.api.Test; + +class AstBuilderTest { + + /** + * SQL syntax parser that helps prepare parse tree as AstBuilder input. + */ + private final SQLSyntaxParser parser = new SQLSyntaxParser(); + + /** + * AST builder class that being tested. + */ + private final AstBuilder astBuilder = new AstBuilder(); + + @Test + public void buildASTForSelectLiterals() { + assertEquals( + project( + values(emptyList()), + intLiteral(123), + stringLiteral("hello"), + booleanLiteral(false), + doubleLiteral(-4.567) + ), + buildAST("SELECT 123, 'hello', false, -4.567") + ); + } + + private UnresolvedPlan buildAST(String query) { + ParseTree parseTree = parser.parse(query); + return parseTree.accept(astBuilder); + } + +} \ No newline at end of file diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilderTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilderTest.java new file mode 100644 index 0000000000..cd304b5d38 --- /dev/null +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilderTest.java @@ -0,0 +1,117 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.sql.parser; + +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.booleanLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.doubleLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.function; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.intLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.stringLiteral; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.amazon.opendistroforelasticsearch.sql.ast.Node; +import com.amazon.opendistroforelasticsearch.sql.common.antlr.CaseInsensitiveCharStream; +import com.amazon.opendistroforelasticsearch.sql.common.antlr.SyntaxAnalysisErrorListener; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLLexer; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser; +import org.antlr.v4.runtime.CommonTokenStream; +import org.junit.jupiter.api.Test; + +class AstExpressionBuilderTest { + + private final AstExpressionBuilder astExprBuilder = new AstExpressionBuilder(); + + @Test + public void canBuildStringLiteral() { + assertEquals( + stringLiteral("hello"), + buildExprAst("'hello'") + ); + } + + @Test + public void canBuildIntegerLiteral() { + assertEquals( + intLiteral(123), + buildExprAst("123") + ); + } + + @Test + public void canBuildNegativeRealLiteral() { + assertEquals( + doubleLiteral(-4.567), + buildExprAst("-4.567") + ); + } + + @Test + public void canBuildBooleanLiteral() { + assertEquals( + booleanLiteral(true), + buildExprAst("true") + ); + } + + @Test + public void canBuildArithmeticExpression() { + assertEquals( + function("+", intLiteral(1), intLiteral(2)), + buildExprAst("1 + 2") + ); + } + + @Test + public void canBuildExpressionWithParentheses() { + assertEquals( + function("*", + function("+", doubleLiteral(-1.0), doubleLiteral(2.3)), + function("-", intLiteral(3), intLiteral(1)) + ), + buildExprAst("(-1.0 + 2.3) * (3 - 1)") + ); + } + + @Test + public void canBuildFunctionCall() { + assertEquals( + function("abs", intLiteral(-1)), + buildExprAst("abs(-1)") + ); + } + + @Test + public void canBuildNestedFunctionCall() { + assertEquals( + function("abs", + function("*", + function("abs", intLiteral(-5)), + intLiteral(-1) + ) + ), + buildExprAst("abs(abs(-5) * -1)") + ); + } + + private Node buildExprAst(String expr) { + OpenDistroSQLLexer lexer = new OpenDistroSQLLexer(new CaseInsensitiveCharStream(expr)); + OpenDistroSQLParser parser = new OpenDistroSQLParser(new CommonTokenStream(lexer)); + parser.addErrorListener(new SyntaxAnalysisErrorListener()); + return parser.expression().accept(astExprBuilder); + } + +} \ No newline at end of file diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SqlPlug.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SqlPlug.java deleted file mode 100644 index 00e657409a..0000000000 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SqlPlug.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.plugin; - -import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; -import com.amazon.opendistroforelasticsearch.sql.executor.AsyncRestExecutor; -import com.amazon.opendistroforelasticsearch.sql.metrics.Metrics; -import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.settings.ClusterSettings; -import org.elasticsearch.common.settings.IndexScopedSettings; -import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.plugins.ActionPlugin; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.rest.RestController; -import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.threadpool.ExecutorBuilder; -import org.elasticsearch.threadpool.FixedExecutorBuilder; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.watcher.ResourceWatcherService; - -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.function.Supplier; - -public class SqlPlug extends Plugin implements ActionPlugin { - - /** - * Sql plugin specific settings in ES cluster settings - */ - private final SqlSettings sqlSettings = new SqlSettings(); - - public SqlPlug() { - } - - - public String name() { - return "sql"; - } - - public String description() { - return "Use sql to query elasticsearch."; - } - - - @Override - public List getRestHandlers(Settings settings, RestController restController, - ClusterSettings clusterSettings, IndexScopedSettings indexScopedSettings, - SettingsFilter settingsFilter, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster) { - LocalClusterState.state().setResolver(indexNameExpressionResolver); - Metrics.getInstance().registerDefaultMetrics(); - return Arrays.asList( - new RestSqlAction(settings, restController), - new RestSqlStatsAction(settings, restController), - new RestSqlSettingsAction(settings, restController)); - } - - @Override - public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver) { - LocalClusterState.state().setClusterService(clusterService); - LocalClusterState.state().setSqlSettings(sqlSettings); - return super.createComponents(client, clusterService, threadPool, resourceWatcherService, scriptService, - xContentRegistry, environment, nodeEnvironment, namedWriteableRegistry, indexNameExpressionResolver); - } - - @Override - public List> getExecutorBuilders(Settings settings) { - return Collections.singletonList( - new FixedExecutorBuilder( - settings, - AsyncRestExecutor.SQL_WORKER_THREAD_POOL_NAME, - EsExecutors.numberOfProcessors(settings), - 1000, - null - ) - ); - } - - @Override - public List> getSettings() { - return sqlSettings.getSettings(); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/CorrectnessIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/CorrectnessIT.java deleted file mode 100644 index c2aab4a6b2..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/CorrectnessIT.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.correctness; - -import com.amazon.opendistroforelasticsearch.sql.correctness.report.TestReport; -import com.amazon.opendistroforelasticsearch.sql.correctness.runner.ComparisonTest; -import com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection.DBConnection; -import com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection.ESConnection; -import com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection.JDBCConnection; -import com.amazon.opendistroforelasticsearch.sql.correctness.testset.TestDataSet; -import com.amazon.opendistroforelasticsearch.sql.esintgtest.CustomExternalTestCluster; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import com.google.common.collect.Maps; -import org.apache.http.HttpHost; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.TestCluster; -import org.json.JSONObject; -import org.junit.Test; - -import java.io.IOException; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Map; -import java.util.TimeZone; - -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getResourceFilePath; - -/** - * Correctness integration test by performing comparison test with other databases. - */ -@ESIntegTestCase.SuiteScopeTestCase -@ESIntegTestCase.ClusterScope(scope=ESIntegTestCase.Scope.SUITE, numDataNodes=3, supportsDedicatedMasters=false, transportClientRatio=1) -@ThreadLeakScope(ThreadLeakScope.Scope.NONE) -public class CorrectnessIT extends ESIntegTestCase { - - private static final Logger LOG = LogManager.getLogger(); - - @Test - public void performComparisonTest() { - TestConfig config = new TestConfig(getCmdLineArgs()); - LOG.info("Starting comparison test {}", config); - - try (ComparisonTest test = new ComparisonTest(getThisDBConnection(config), - getOtherDBConnections(config))) { - LOG.info("Loading test data set..."); - test.connect(); - for (TestDataSet dataSet : config.getTestDataSets()) { - test.loadData(dataSet); - } - - LOG.info("Verifying test queries..."); - TestReport report = test.verify(config.getTestQuerySet()); - - LOG.info("Saving test report to disk..."); - store(report); - - LOG.info("Cleaning up test data..."); - for (TestDataSet dataSet : config.getTestDataSets()) { - test.cleanUp(dataSet); - } - } - LOG.info("Completed comparison test."); - } - - private Map getCmdLineArgs() { - return Maps.fromProperties(System.getProperties()); - } - - private DBConnection getThisDBConnection(TestConfig config) { - String dbUrl = config.getDbConnectionUrl(); - if (dbUrl.isEmpty()) { - return getESConnection(config); - } - return new JDBCConnection("DB Tested", dbUrl); - } - - /** Use Elasticsearch cluster given on CLI arg or internal embedded in SQLIntegTestCase */ - private DBConnection getESConnection(TestConfig config) { - RestClient client; - String esHost = config.getESHostUrl(); - if (esHost.isEmpty()) { - client = getRestClient(); - esHost = client.getNodes().get(0).getHost().toString(); - } else { - client = RestClient.builder(HttpHost.create(esHost)).build(); - } - return new ESConnection("jdbc:elasticsearch://" + esHost, client); - } - - /** Create database connection with database name and connect URL */ - private DBConnection[] getOtherDBConnections(TestConfig config) { - return config.getOtherDbConnectionNameAndUrls(). - entrySet().stream(). - map(e -> new JDBCConnection(e.getKey(), e.getValue())). - toArray(DBConnection[]::new); - } - - private void store(TestReport report) { - try { - // Create reports folder if not exists - String folderPath = "reports/"; - Path path = Paths.get(getResourceFilePath(folderPath)); - if (Files.notExists(path)) { - Files.createDirectory(path); - } - - // Write to report file - String relFilePath = folderPath + reportFileName(); - String absFilePath = getResourceFilePath(relFilePath); - byte[] content = new JSONObject(report).toString(2).getBytes(); - - LOG.info("Report file location is {}", absFilePath); - Files.write(Paths.get(absFilePath), content); - } catch (Exception e) { - throw new IllegalStateException("Failed to store report file", e); - } - } - - private String reportFileName() { - SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd-HH"); - df.setTimeZone(TimeZone.getTimeZone("GMT")); - String dateTime = df.format(new Date()); - return "report_" + dateTime + ".json"; - } - - @Override - protected TestCluster buildTestCluster(Scope scope, long seed) throws IOException { - - String clusterAddresses = System.getProperty(TESTS_CLUSTER); - - if (Strings.hasLength(clusterAddresses)) { - String[] stringAddresses = clusterAddresses.split(","); - TransportAddress[] transportAddresses = new TransportAddress[stringAddresses.length]; - int i = 0; - for (String stringAddress : stringAddresses) { - URL url = new URL("http://" + stringAddress); - InetAddress inetAddress = InetAddress.getByName(url.getHost()); - transportAddresses[i++] = new TransportAddress(new InetSocketAddress(inetAddress, url.getPort())); - } - return new CustomExternalTestCluster(createTempDir(), externalClusterClientSettings(), - transportClientPlugins(), transportAddresses); - } - return super.buildTestCluster(scope, seed); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/TestConfig.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/TestConfig.java deleted file mode 100644 index 3d55a00b6d..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/TestConfig.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.correctness; - -import com.amazon.opendistroforelasticsearch.sql.correctness.testset.TestDataSet; -import com.amazon.opendistroforelasticsearch.sql.correctness.testset.TestQuerySet; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; -import com.google.common.base.Charsets; -import com.google.common.io.Resources; - -import java.net.URL; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; - -import static java.util.stream.Collectors.joining; - -/** - * Test configuration parse the following information from command line arguments: - * 1) Test schema and data - * 2) Test queries - * 3) Elasticsearch connection URL - * 4) Other database connection URLs - */ -public class TestConfig { - - private static final String DEFAULT_TEST_QUERIES = "tableau_integration_tests.txt"; - private static final String DEFAULT_OTHER_DB_URLS = "H2=jdbc:h2:mem:test;DB_CLOSE_DELAY=-1," + "SQLite=jdbc:sqlite::memory:"; - - private final TestDataSet[] testDataSets; - - private final TestQuerySet testQuerySet; - - private final String esHostUrl; - - /** Test against some database rather than Elasticsearch via our JDBC driver */ - private final String dbConnectionUrl; - - private final Map otherDbConnectionNameAndUrls = new HashMap<>(); - - public TestConfig(Map cliArgs) { - testDataSets = buildDefaultTestDataSet(); // TODO: parse test data set argument - testQuerySet = buildTestQuerySet(cliArgs); - esHostUrl = cliArgs.getOrDefault("esHost", ""); - dbConnectionUrl = cliArgs.getOrDefault("dbUrl", ""); - - parseOtherDbConnectionInfo(cliArgs); - } - - public TestDataSet[] getTestDataSets() { - return testDataSets; - } - - public TestQuerySet getTestQuerySet() { - return testQuerySet; - } - - public String getESHostUrl() { - return esHostUrl; - } - - public String getDbConnectionUrl() { - return dbConnectionUrl; - } - - public Map getOtherDbConnectionNameAndUrls() { - return otherDbConnectionNameAndUrls; - } - - private TestDataSet[] buildDefaultTestDataSet() { - return new TestDataSet[]{ - new TestDataSet("kibana_sample_data_flights", - readFile("kibana_sample_data_flights.json"), - readFile("kibana_sample_data_flights.csv")), - new TestDataSet("kibana_sample_data_ecommerce", - readFile("kibana_sample_data_ecommerce.json"), - readFile("kibana_sample_data_ecommerce.csv")), - }; - } - - private TestQuerySet buildTestQuerySet(Map cliArgs) { - String queryFilePath = cliArgs.getOrDefault("queries", ""); // Gradle set it empty always - if (queryFilePath.isEmpty()) { - queryFilePath = DEFAULT_TEST_QUERIES; - } - return new TestQuerySet(readFile(queryFilePath)); - } - - private void parseOtherDbConnectionInfo(Map cliArgs) { - String otherDbUrls = cliArgs.getOrDefault("otherDbUrls", ""); - if (otherDbUrls.isEmpty()) { - otherDbUrls = DEFAULT_OTHER_DB_URLS; - } - - for (String dbNameAndUrl : otherDbUrls.split(",")) { - int firstEq = dbNameAndUrl.indexOf('='); - String dbName = dbNameAndUrl.substring(0, firstEq); - String dbUrl = dbNameAndUrl.substring(firstEq + 1); - otherDbConnectionNameAndUrls.put(dbName, dbUrl); - } - } - - private static String readFile(String relativePath) { - try { - URL url = Resources.getResource("correctness/" + relativePath); - return Resources.toString(url, Charsets.UTF_8); - } catch (Exception e) { - throw new IllegalStateException("Failed to read test file [" + relativePath + "]"); - } - } - - @Override - public String toString() { - return "\n=================================\n" - + "Tested Database : " + esHostUrlToString() + '\n' - + "Other Databases :\n" + otherDbConnectionInfoToString() + '\n' - + "Test data set(s) :\n" + testDataSetsToString() + '\n' - + "Test query set : " + testQuerySet + '\n' - + "=================================\n"; - } - - private String testDataSetsToString() { - return Arrays.stream(testDataSets). - map(TestDataSet::toString). - collect(joining("\n")); - } - - private String esHostUrlToString() { - if (!dbConnectionUrl.isEmpty()) { - return dbConnectionUrl; - } - return esHostUrl.isEmpty() ? "(Use internal Elasticsearch in workspace)" : esHostUrl; - } - - private String otherDbConnectionInfoToString() { - return otherDbConnectionNameAndUrls.entrySet().stream(). - map(e -> StringUtils.format(" %s = %s", e.getKey(), e.getValue())). - collect(joining("\n")); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/ComparisonTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/ComparisonTest.java deleted file mode 100644 index a27bfd5142..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/ComparisonTest.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.correctness.runner; - -import com.amazon.opendistroforelasticsearch.sql.correctness.report.ErrorTestCase; -import com.amazon.opendistroforelasticsearch.sql.correctness.report.FailedTestCase; -import com.amazon.opendistroforelasticsearch.sql.correctness.report.SuccessTestCase; -import com.amazon.opendistroforelasticsearch.sql.correctness.report.TestCaseReport; -import com.amazon.opendistroforelasticsearch.sql.correctness.report.TestReport; -import com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection.DBConnection; -import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.DBResult; -import com.amazon.opendistroforelasticsearch.sql.correctness.testset.TestDataSet; -import com.amazon.opendistroforelasticsearch.sql.correctness.testset.TestQuerySet; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; -import com.google.common.collect.Iterators; -import com.google.common.collect.Lists; - -import java.util.Arrays; -import java.util.Comparator; -import java.util.Iterator; -import java.util.List; - -import static com.google.common.collect.ObjectArrays.concat; - -/** - * Comparison test runner for query result correctness. - */ -public class ComparisonTest implements AutoCloseable { - - /** Next id for test case */ - private int testCaseId = 1; - - /** Connection for database being tested */ - private final DBConnection thisConnection; - - /** Database connections for reference databases */ - private final DBConnection[] otherDbConnections; - - public ComparisonTest(DBConnection thisConnection, DBConnection[] otherDbConnections) { - this.thisConnection = thisConnection; - this.otherDbConnections = otherDbConnections; - - // Guarantee ordering of other database in comparison test - Arrays.sort(this.otherDbConnections, Comparator.comparing(DBConnection::getDatabaseName)); - } - - /** - * Open database connection. - */ - public void connect() { - for (DBConnection conn : concat(thisConnection, otherDbConnections)) { - conn.connect(); - } - } - - /** - * Create table and load test data. - * @param dataSet test data set - */ - public void loadData(TestDataSet dataSet) { - for (DBConnection conn : concat(thisConnection, otherDbConnections)) { - conn.create(dataSet.getTableName(), dataSet.getSchema()); - insertTestDataInBatch(conn, dataSet.getTableName(), dataSet.getDataRows()); - } - } - - /** - * Verify queries one by one by comparing between databases. - * @param querySet SQL queries - * @return Test result report - */ - public TestReport verify(TestQuerySet querySet) { - TestReport report = new TestReport(); - for (String sql : querySet) { - try { - DBResult esResult = thisConnection.select(sql); - report.addTestCase(compareWithOtherDb(sql, esResult)); - } catch (Exception e) { - report.addTestCase(new ErrorTestCase(nextId(), sql, - StringUtils.format("%s: %s", e.getClass().getSimpleName(), extractRootCause(e)))); - } - } - return report; - } - - /** - * Clean up test table. - * @param dataSet test data set - */ - public void cleanUp(TestDataSet dataSet) { - for (DBConnection conn : concat(thisConnection, otherDbConnections)) { - conn.drop(dataSet.getTableName()); - } - } - - @Override - public void close() { - for (DBConnection conn : concat(thisConnection, otherDbConnections)) { - try { - conn.close(); - } catch (Exception e) { - // Ignore - } - } - } - - /** Execute the query and compare with current result */ - private TestCaseReport compareWithOtherDb(String sql, DBResult esResult) { - List mismatchResults = Lists.newArrayList(esResult); - StringBuilder reasons = new StringBuilder(); - for (int i = 0; i < otherDbConnections.length; i++) { - try { - DBResult otherDbResult = otherDbConnections[i].select(sql); - if (esResult.equals(otherDbResult)) { - return new SuccessTestCase(nextId(), sql); - } - - mismatchResults.add(otherDbResult); - - // Cannot find any database result match - if (i == otherDbConnections.length - 1) { - return new FailedTestCase(nextId(), sql, mismatchResults); - } - } catch (Exception e) { - // Ignore and move on to next database - reasons.append(extractRootCause(e)).append(";"); - } - } - - // Cannot find any database support this query - return new ErrorTestCase(nextId(), sql, "No other databases support this query: " + reasons); - } - - private int nextId() { - return testCaseId++; - } - - private void insertTestDataInBatch(DBConnection conn, String tableName, List testData) { - Iterator iterator = testData.iterator(); - String[] fieldNames = iterator.next(); // first row is header of column names - Iterators.partition(iterator, 100). - forEachRemaining(batch -> conn.insert(tableName, fieldNames, batch)); - } - - private String extractRootCause(Throwable e) { - while (e.getCause() != null) { - e = e.getCause(); - } - - if (e.getLocalizedMessage() != null) { - return e.getLocalizedMessage(); - } - if (e.getMessage() != null) { - return e.getMessage(); - } - return e.toString(); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/DBConnection.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/DBConnection.java deleted file mode 100644 index df2174d471..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/DBConnection.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection; - -import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.DBResult; - -import java.util.List; - -/** - * Abstraction for different databases. - */ -public interface DBConnection { - - /** - * @return database name - */ - String getDatabaseName(); - - /** - * Connect to database by opening a connection. - */ - void connect(); - - /** - * Create table with the schema. - * @param tableName table name - * @param schema schema json in ES mapping format - */ - void create(String tableName, String schema); - - /** - * Insert batch of data to database. - * @param tableName table name - * @param columnNames column names - * @param batch batch of rows - */ - void insert(String tableName, String[] columnNames, List batch); - - /** - * Fetch data from database. - * @param query SQL query - * @return result set - */ - DBResult select(String query); - - /** - * Drop table. - * @param tableName table name - */ - void drop(String tableName); - - /** - * Close the database connection. - */ - void close(); - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/ESConnection.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/ESConnection.java deleted file mode 100644 index 17fe6afd8b..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/ESConnection.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection; - -import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.DBResult; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.RestClient; -import org.json.JSONObject; - -import java.io.IOException; -import java.util.List; - -/** - * Elasticsearch database connection for insertion. This class wraps JDBCConnection to delegate query method. - */ -public class ESConnection implements DBConnection { - - /** Connection via our Elasticsearch JDBC driver */ - private final DBConnection connection; - - /** Native Elasticsearch REST client for operation unsupported by driver such as CREATE/INSERT */ - private final RestClient client; - - public ESConnection(String connectionUrl, RestClient client) { - this.connection = new JDBCConnection("Elasticsearch", connectionUrl); - this.client = client; - } - - @Override - public String getDatabaseName() { - return "Elasticsearch"; - } - - @Override - public void connect() { - connection.connect(); - } - - @Override - public void create(String tableName, String schema) { - Request request = new Request("PUT", "/" + tableName); - request.setJsonEntity(schema); - performRequest(request); - } - - @Override - public void drop(String tableName) { - performRequest(new Request("DELETE", "/" + tableName)); - } - - @Override - public void insert(String tableName, String[] columnNames, List batch) { - Request request = new Request("POST", "/" + tableName + "/_bulk"); - request.setJsonEntity(buildBulkBody(columnNames, batch)); - performRequest(request); - } - - @Override - public DBResult select(String query) { - return connection.select(query); - } - - @Override - public void close() { - connection.close(); - try { - client.close(); - } catch (IOException e) { - // Ignore - } - } - - private void performRequest(Request request) { - try { - Response response = client.performRequest(request); - int status = response.getStatusLine().getStatusCode(); - if (status != 200) { - throw new IllegalStateException("Failed to perform request. Error code: " + status); - } - } catch (IOException e) { - throw new IllegalStateException("Failed to perform request", e); - } - } - - private String buildBulkBody(String[] columnNames, List batch) { - StringBuilder body = new StringBuilder(); - for (String[] fieldValues : batch) { - JSONObject json = new JSONObject(); - for (int i = 0; i < columnNames.length; i++) { - json.put(columnNames[i], fieldValues[i]); - } - - body.append("{\"index\":{}}\n"). - append(json).append("\n"); - } - return body.toString(); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/JDBCConnection.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/JDBCConnection.java deleted file mode 100644 index 7519a4e0f6..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/JDBCConnection.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection; - -import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.DBResult; -import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.Row; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; -import com.google.common.base.Strings; -import org.json.JSONObject; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Arrays; -import java.util.List; - -import static java.util.stream.Collectors.joining; - -/** - * Database connection by JDBC driver. - */ -public class JDBCConnection implements DBConnection { - - private static final String SINGLE_QUOTE = "'"; - private static final String DOUBLE_QUOTE = "''"; - - /** Database name for display */ - private final String databaseName; - - /** Database connection URL */ - private final String connectionUrl; - - /** Current live connection */ - private Connection connection; - - public JDBCConnection(String databaseName, String connectionUrl) { - this.databaseName = databaseName; - this.connectionUrl = connectionUrl; - } - - @Override - public void connect() { - try { - connection = DriverManager.getConnection(connectionUrl); - } catch (Exception e) { - throw new IllegalStateException("Failed to open connection", e); - } - } - - @Override - public String getDatabaseName() { - return databaseName; - } - - @Override - public void create(String tableName, String schema) { - try (Statement stmt = connection.createStatement()) { - String types = parseColumnNameAndTypesInSchemaJson(schema); - stmt.executeUpdate(StringUtils.format("CREATE TABLE %s(%s)", tableName, types)); - } catch (SQLException e) { - throw new IllegalStateException("Failed to create table [" + tableName + "]", e); - } - } - - @Override - public void drop(String tableName) { - try (Statement stmt = connection.createStatement()) { - stmt.executeUpdate("DROP TABLE " + tableName); - } catch (SQLException e) { - throw new IllegalStateException("Failed to drop table [" + tableName + "]", e); - } - } - - @Override - public void insert(String tableName, String[] columnNames, List batch) { - try (Statement stmt = connection.createStatement()) { - String names = String.join(",", columnNames); - for (String[] fieldValues : batch) { - stmt.addBatch(StringUtils.format( - "INSERT INTO %s(%s) VALUES (%s)", tableName, names, getValueList(fieldValues))); - } - stmt.executeBatch(); - } catch (SQLException e) { - throw new IllegalStateException("Failed to execute update", e); - } - } - - @Override - public DBResult select(String query) { - try (Statement stmt = connection.createStatement()) { - ResultSet resultSet = stmt.executeQuery(query); - DBResult result = new DBResult(databaseName); - populateMetaData(resultSet, result); - populateData(resultSet, result); - return result; - } catch (SQLException e) { - throw new IllegalStateException("Failed to execute query [" + query + "]", e); - } - } - - @Override - public void close() { - try { - connection.close(); - } catch (SQLException e) { - // Ignore - } - } - - /** Parse out type in schema json and convert to field name and type pairs for CREATE TABLE statement. */ - private String parseColumnNameAndTypesInSchemaJson(String schema) { - JSONObject json = (JSONObject) new JSONObject(schema).query("/mappings/properties"); - return json.keySet().stream(). - map(colName -> colName + " " + mapToJDBCType(json.getJSONObject(colName).getString("type"))). - collect(joining(",")); - } - - private String getValueList(String[] fieldValues) { - return Arrays.stream(fieldValues). - map(val -> val.replace(SINGLE_QUOTE, DOUBLE_QUOTE)). - map(val -> SINGLE_QUOTE + val + SINGLE_QUOTE). - collect(joining(",")); - } - - private void populateMetaData(ResultSet resultSet, DBResult result) throws SQLException { - ResultSetMetaData metaData = resultSet.getMetaData(); - for (int i = 1; i <= metaData.getColumnCount(); i++) { - - // Use label name (alias) if present. - String colName = metaData.getColumnLabel(i); - if (Strings.isNullOrEmpty(colName)) { - colName = metaData.getColumnName(i); - } - result.addColumn(colName, metaData.getColumnTypeName(i)); - } - } - - private void populateData(ResultSet resultSet, DBResult result) throws SQLException { - while (resultSet.next()) { - Row row = new Row(); - for (int i = 1; i <= result.columnSize(); i++) { - row.add(resultSet.getObject(i)); - } - result.addRow(row); - } - } - - private String mapToJDBCType(String esType) { - switch (esType.toUpperCase()) { - case "KEYWORD": - case "TEXT": - return "VARCHAR"; - case "DATE": - return "TIMESTAMP"; - case "HALF_FLOAT": - return "FLOAT"; - default: - return esType; - } - } - - /** Setter for unit test mock */ - public void setConnection(Connection connection) { - this.connection = connection; - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/DBResult.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/DBResult.java deleted file mode 100644 index 9c47fa5c19..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/DBResult.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset; - -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; -import com.google.common.collect.ImmutableSet; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.ToString; -import org.json.JSONPropertyName; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Query result for equality comparison. Based on different type of query, such as query with/without ORDER BY and - * query with SELECT columns or just *, order of column and row may matter or not. So the internal data structure of this - * class is passed in from outside either list or set, hash map or linked hash map etc. - */ -@EqualsAndHashCode(exclude = "databaseName") -@ToString -public class DBResult { - - /** Possible types for floating point number */ - private static final Set FLOAT_TYPES = ImmutableSet.of("FLOAT", "DOUBLE", "REAL"); - - /** Database name for display */ - private final String databaseName; - - /** Column name and types from result set meta data */ - @Getter - private final Collection schema; - - /** Data rows from result set */ - private final Collection dataRows; - - /** - * By default treat both columns and data rows in order. This makes sense for typical query - * with specific column names in SELECT but without ORDER BY. - */ - public DBResult(String databaseName) { - this(databaseName, new ArrayList<>(), new HashSet<>()); - } - - public DBResult(String databaseName, Collection schema, Collection rows) { - this.databaseName = databaseName; - this.schema = schema; - this.dataRows = rows; - } - - public int columnSize() { - return schema.size(); - } - - public void addColumn(String name, String type) { - type = StringUtils.toUpper(type); - - // Ignore float type by assigning all type names string to it. - if (FLOAT_TYPES.contains(type)) { - type = FLOAT_TYPES.toString(); - } - schema.add(new Type(StringUtils.toUpper(name), type)); - } - - public void addRow(Row row) { - dataRows.add(row); - } - - @JSONPropertyName("database") - public String getDatabaseName() { - return databaseName; - } - - /** Flatten for simplifying json generated */ - public Collection> getDataRows() { - return dataRows.stream().map(Row::getValues).collect(Collectors.toSet()); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/Row.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/Row.java deleted file mode 100644 index 3f83d8b4e6..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/Row.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset; - -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.ToString; - -import java.math.BigDecimal; -import java.math.RoundingMode; -import java.util.ArrayList; -import java.util.Collection; - -/** - * Row in result set. - */ -@EqualsAndHashCode -@ToString -@Getter -public class Row { - - private final Collection values; - - public Row() { - this(new ArrayList<>()); // values in order by default - } - - public Row(Collection values) { - this.values = values; - } - - public void add(Object value) { - values.add(roundFloatNum(value)); - } - - private Object roundFloatNum(Object value) { - if (value instanceof Float) { - BigDecimal decimal = BigDecimal.valueOf((Float) value).setScale(2, RoundingMode.CEILING); - value = decimal.doubleValue(); // Convert to double too - } else if (value instanceof Double) { - BigDecimal decimal = BigDecimal.valueOf((Double) value).setScale(2, RoundingMode.CEILING); - value = decimal.doubleValue(); - } - return value; - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/ComparisonTestTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/ComparisonTestTest.java deleted file mode 100644 index 5630b80873..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/ComparisonTestTest.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.correctness.tests; - -import com.amazon.opendistroforelasticsearch.sql.correctness.report.ErrorTestCase; -import com.amazon.opendistroforelasticsearch.sql.correctness.report.FailedTestCase; -import com.amazon.opendistroforelasticsearch.sql.correctness.report.SuccessTestCase; -import com.amazon.opendistroforelasticsearch.sql.correctness.report.TestReport; -import com.amazon.opendistroforelasticsearch.sql.correctness.runner.ComparisonTest; -import com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection.DBConnection; -import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.DBResult; -import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.Row; -import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.Type; -import com.amazon.opendistroforelasticsearch.sql.correctness.testset.TestQuerySet; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.runners.MockitoJUnitRunner; - -import static java.util.Arrays.asList; -import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * Tests for {@link ComparisonTest} - */ -@RunWith(MockitoJUnitRunner.class) -public class ComparisonTestTest { - - @Mock - private DBConnection esConnection; - - @Mock - private DBConnection otherDbConnection; - - private ComparisonTest correctnessTest; - - @Before - public void setUp() { - when(esConnection.getDatabaseName()).thenReturn("ES"); - when(otherDbConnection.getDatabaseName()).thenReturn("Other"); - correctnessTest = new ComparisonTest( - esConnection, new DBConnection[]{otherDbConnection} - ); - } - - @Test - public void testSuccess() { - when(esConnection.select(anyString())).thenReturn( - new DBResult("ES", asList(new Type("firstname", "text")), asList(new Row(asList("John")))) - ); - when(otherDbConnection.select(anyString())).thenReturn( - new DBResult("Other DB", asList(new Type("firstname", "text")), asList(new Row(asList("John")))) - ); - - TestReport expected = new TestReport(); - expected.addTestCase(new SuccessTestCase(1, "SELECT * FROM accounts")); - TestReport actual = correctnessTest.verify(querySet("SELECT * FROM accounts")); - assertEquals(expected, actual); - } - - @Test - public void testFailureDueToInconsistency() { - DBResult esResult = new DBResult("ES", asList(new Type("firstname", "text")), asList(new Row(asList("John")))); - DBResult otherDbResult = new DBResult("Other DB", asList(new Type("firstname", "text")), asList(new Row(asList("JOHN")))); - when(esConnection.select(anyString())).thenReturn(esResult); - when(otherDbConnection.select(anyString())).thenReturn(otherDbResult); - - TestReport expected = new TestReport(); - expected.addTestCase(new FailedTestCase(1, "SELECT * FROM accounts", asList(esResult, otherDbResult))); - TestReport actual = correctnessTest.verify(querySet("SELECT * FROM accounts")); - assertEquals(expected, actual); - } - - @Test - public void testSuccessFinally() { - DBConnection anotherDbConnection = mock(DBConnection.class); - when(anotherDbConnection.getDatabaseName()).thenReturn("Another"); - correctnessTest = new ComparisonTest( - esConnection, new DBConnection[]{otherDbConnection, anotherDbConnection} - ); - - DBResult esResult = new DBResult("ES", asList(new Type("firstname", "text")), asList(new Row(asList("John")))); - DBResult otherDbResult = new DBResult("Other DB", asList(new Type("firstname", "text")), asList(new Row(asList("JOHN")))); - DBResult anotherDbResult = new DBResult("Another DB", asList(new Type("firstname", "text")), asList(new Row(asList("John")))); - when(esConnection.select(anyString())).thenReturn(esResult); - when(otherDbConnection.select(anyString())).thenReturn(otherDbResult); - when(anotherDbConnection.select(anyString())).thenReturn(anotherDbResult); - - TestReport expected = new TestReport(); - expected.addTestCase(new SuccessTestCase(1, "SELECT * FROM accounts")); - TestReport actual = correctnessTest.verify(querySet("SELECT * FROM accounts")); - assertEquals(expected, actual); - } - - @Test - public void testFailureDueToEventualInconsistency() { - DBConnection anotherDbConnection = mock(DBConnection.class); - when(anotherDbConnection.getDatabaseName()).thenReturn("ZZZ DB"); // Make sure this will be called after Other DB - correctnessTest = new ComparisonTest( - esConnection, new DBConnection[]{otherDbConnection, anotherDbConnection} - ); - - DBResult esResult = new DBResult("ES", asList(new Type("firstname", "text")), asList(new Row(asList("John")))); - DBResult otherDbResult = new DBResult("Other DB", asList(new Type("firstname", "text")), asList(new Row(asList("JOHN")))); - DBResult anotherDbResult = new DBResult("ZZZ DB", asList(new Type("firstname", "text")), asList(new Row(asList("Hank")))); - when(esConnection.select(anyString())).thenReturn(esResult); - when(otherDbConnection.select(anyString())).thenReturn(otherDbResult); - when(anotherDbConnection.select(anyString())).thenReturn(anotherDbResult); - - TestReport expected = new TestReport(); - expected.addTestCase(new FailedTestCase(1, "SELECT * FROM accounts", asList(esResult, otherDbResult, anotherDbResult))); - TestReport actual = correctnessTest.verify(querySet("SELECT * FROM accounts")); - assertEquals(expected, actual); - } - - @Test - public void testErrorDueToESException() { - when(esConnection.select(anyString())).thenThrow(new RuntimeException("All shards failure")); - - TestReport expected = new TestReport(); - expected.addTestCase(new ErrorTestCase(1, "SELECT * FROM accounts", "RuntimeException: All shards failure")); - TestReport actual = correctnessTest.verify(querySet("SELECT * FROM accounts")); - assertEquals(expected, actual); - } - - @Test - public void testErrorDueToNoOtherDBSupportThisQuery() { - when(esConnection.select(anyString())).thenReturn( - new DBResult("ES", asList(new Type("firstname", "text")), asList(new Row(asList("John")))) - ); - when(otherDbConnection.select(anyString())).thenThrow(new RuntimeException("Unsupported feature")); - - TestReport expected = new TestReport(); - expected.addTestCase(new ErrorTestCase(1, "SELECT * FROM accounts", "No other databases support this query: Unsupported feature;")); - TestReport actual = correctnessTest.verify(querySet("SELECT * FROM accounts")); - assertEquals(expected, actual); - } - - @Test - public void testSuccessWhenOneDBSupportThisQuery() { - DBConnection anotherDbConnection = mock(DBConnection.class); - when(anotherDbConnection.getDatabaseName()).thenReturn("Another"); - correctnessTest = new ComparisonTest( - esConnection, new DBConnection[]{otherDbConnection, anotherDbConnection} - ); - - when(esConnection.select(anyString())).thenReturn( - new DBResult("ES", asList(new Type("firstname", "text")), asList(new Row(asList("John")))) - ); - when(otherDbConnection.select(anyString())).thenThrow(new RuntimeException("Unsupported feature")); - when(anotherDbConnection.select(anyString())).thenReturn( - new DBResult("Another DB", asList(new Type("firstname", "text")), asList(new Row(asList("John")))) - ); - - TestReport expected = new TestReport(); - expected.addTestCase(new SuccessTestCase(1, "SELECT * FROM accounts")); - TestReport actual = correctnessTest.verify(querySet("SELECT * FROM accounts")); - assertEquals(expected, actual); - } - - private TestQuerySet querySet(String query) { - return new TestQuerySet(new String[]{ query }); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/DBResultTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/DBResultTest.java deleted file mode 100644 index c71f996599..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/DBResultTest.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.correctness.tests; - -import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.DBResult; -import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.Type; -import org.junit.Test; - -import java.util.Arrays; - -import static java.util.Collections.emptyList; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; - -/** - * Unit tests for {@link DBResult} - */ -public class DBResultTest { - - @Test - public void dbResultFromDifferentDbNameShouldEqual() { - DBResult result1 = new DBResult("DB 1", Arrays.asList(new Type("name", "VARCHAR")), emptyList()); - DBResult result2 = new DBResult("DB 2", Arrays.asList(new Type("name", "VARCHAR")), emptyList()); - assertEquals(result1, result2); - } - - @Test - public void dbResultWithDifferentColumnShouldNotEqual() { - DBResult result1 = new DBResult("DB 1", Arrays.asList(new Type("name", "VARCHAR")), emptyList()); - DBResult result2 = new DBResult("DB 2", Arrays.asList(new Type("age", "INT")), emptyList()); - assertNotEquals(result1, result2); - } - - @Test - public void dbResultWithDifferentColumnTypeShouldNotEqual() { - DBResult result1 = new DBResult("DB 1", Arrays.asList(new Type("age", "FLOAT")), emptyList()); - DBResult result2 = new DBResult("DB 2", Arrays.asList(new Type("age", "INT")), emptyList()); - assertNotEquals(result1, result2); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/ESConnectionTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/ESConnectionTest.java deleted file mode 100644 index aa4f10a6f9..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/ESConnectionTest.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.correctness.tests; - -import com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection.ESConnection; -import com.google.common.io.CharStreams; -import org.apache.http.ProtocolVersion; -import org.apache.http.message.BasicStatusLine; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.RestClient; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.ArgumentCaptor; -import org.mockito.Mock; -import org.mockito.runners.MockitoJUnitRunner; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.Arrays; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * Tests for {@link ESConnection} - */ -@RunWith(MockitoJUnitRunner.class) -public class ESConnectionTest { - - @Mock - private RestClient client; - - private ESConnection conn; - - @Before - public void setUp() throws IOException { - conn = new ESConnection("jdbc:elasticsearch://localhost:12345", client); - - Response response = mock(Response.class); - when(client.performRequest(any(Request.class))).thenReturn(response); - when(response.getStatusLine()).thenReturn(new BasicStatusLine(new ProtocolVersion("HTTP", 2, 0), 200, "")); - } - - @Test - public void testCreateTable() throws IOException { - conn.create("test", "mapping"); - - Request actual = captureActualArg(); - assertEquals("PUT", actual.getMethod()); - assertEquals("/test", actual.getEndpoint()); - assertEquals("mapping", getBody(actual)); - } - - @Test - public void testInsertData() throws IOException { - conn.insert("test", new String[]{"name"}, Arrays.asList(new String[]{"John"}, new String[]{"Hank"})); - - Request actual = captureActualArg(); - assertEquals("POST", actual.getMethod()); - assertEquals("/test/_bulk", actual.getEndpoint()); - assertEquals( - "{\"index\":{}}\n" - + "{\"name\":\"John\"}\n" - + "{\"index\":{}}\n" - + "{\"name\":\"Hank\"}\n", - getBody(actual) - ); - } - - @Test - public void testDropTable() throws IOException { - conn.drop("test"); - - Request actual = captureActualArg(); - assertEquals("DELETE", actual.getMethod()); - assertEquals("/test", actual.getEndpoint()); - } - - private Request captureActualArg() throws IOException { - ArgumentCaptor argCap = ArgumentCaptor.forClass(Request.class); - verify(client).performRequest(argCap.capture()); - return argCap.getValue(); - } - - private String getBody(Request request) throws IOException { - InputStream inputStream = request.getEntity().getContent(); - return CharStreams.toString(new InputStreamReader(inputStream)); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/JDBCConnectionTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/JDBCConnectionTest.java deleted file mode 100644 index bb2cfd32bf..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/JDBCConnectionTest.java +++ /dev/null @@ -1,221 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.correctness.tests; - -import com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection.JDBCConnection; -import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.DBResult; -import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.Type; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Sets; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.ArgumentCaptor; -import org.mockito.Mock; -import org.mockito.runners.MockitoJUnitRunner; -import org.mockito.stubbing.OngoingStubbing; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.anyInt; -import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * Tests for {@link JDBCConnection} - */ -@RunWith(MockitoJUnitRunner.class) -public class JDBCConnectionTest { - - @Mock - private Connection connection; - - @Mock - private Statement statement; - - private JDBCConnection conn; - - @Before - public void setUp() throws SQLException { - conn = new JDBCConnection("Test DB", "jdbc:testdb://localhost:12345"); - conn.setConnection(connection); - - when(connection.createStatement()).thenReturn(statement); - } - - @Test - public void testCreateTable() throws SQLException { - conn.create("test", "{\"mappings\":{\"properties\":{\"name\":{\"type\":\"keyword\"},\"age\":{\"type\":\"INT\"}}}}"); - - ArgumentCaptor argCap = ArgumentCaptor.forClass(String.class); - verify(statement).executeUpdate(argCap.capture()); - String actual = argCap.getValue(); - - assertEquals("CREATE TABLE test(name VARCHAR,age INT)", actual); - } - - @Test - public void testDropTable() throws SQLException { - conn.drop("test"); - - ArgumentCaptor argCap = ArgumentCaptor.forClass(String.class); - verify(statement).executeUpdate(argCap.capture()); - String actual = argCap.getValue(); - - assertEquals("DROP TABLE test", actual); - } - - @Test - public void testInsertData() throws SQLException { - conn.insert("test", new String[]{"name", "age"}, - Arrays.asList(new String[]{"John", "25"}, new String[]{"Hank", "30"})); - - ArgumentCaptor argCap = ArgumentCaptor.forClass(String.class); - verify(statement, times(2)).addBatch(argCap.capture()); - List actual = argCap.getAllValues(); - - assertEquals( - Arrays.asList( - "INSERT INTO test(name,age) VALUES ('John','25')", - "INSERT INTO test(name,age) VALUES ('Hank','30')" - ), actual - ); - } - - @Test - public void testSelectQuery() throws SQLException { - ResultSetMetaData metaData = mockMetaData(ImmutableMap.of("name", "VARCHAR", "age", "INT")); - ResultSet resultSet = mockResultSet(new Object[]{"John", 25}, new Object[]{"Hank", 30}); - when(statement.executeQuery(anyString())).thenReturn(resultSet); - when(resultSet.getMetaData()).thenReturn(metaData); - - DBResult result = conn.select("SELECT * FROM test"); - assertEquals("Test DB", result.getDatabaseName()); - assertEquals( - Arrays.asList( - new Type("NAME", "VARCHAR"), - new Type("AGE", "INT") - ), - result.getSchema() - ); - assertEquals( - Sets.newHashSet( - Arrays.asList("John", 25), - Arrays.asList("Hank", 30) - ), - result.getDataRows() - ); - } - - @Test - public void testSelectQueryWithAlias() throws SQLException { - ResultSetMetaData metaData = mockMetaData(ImmutableMap.of("name", "VARCHAR", "age", "INT"), "n", "a"); - ResultSet resultSet = mockResultSet(new Object[]{"John", 25}, new Object[]{"Hank", 30}); - when(statement.executeQuery(anyString())).thenReturn(resultSet); - when(resultSet.getMetaData()).thenReturn(metaData); - - DBResult result = conn.select("SELECT * FROM test"); - assertEquals( - Arrays.asList( - new Type("N", "VARCHAR"), - new Type("A", "INT") - ), - result.getSchema() - ); - } - - @Test - public void testSelectQueryWithFloatInResultSet() throws SQLException { - ResultSetMetaData metaData = mockMetaData(ImmutableMap.of("name", "VARCHAR", "balance", "FLOAT")); - ResultSet resultSet = mockResultSet( - new Object[]{"John", 25.123}, - new Object[]{"Hank", 30.456}, - new Object[]{"Allen", 15.1} - ); - when(statement.executeQuery(anyString())).thenReturn(resultSet); - when(resultSet.getMetaData()).thenReturn(metaData); - - DBResult result = conn.select("SELECT * FROM test"); - assertEquals( - Arrays.asList( - new Type("NAME", "VARCHAR"), - new Type("BALANCE", "[FLOAT, DOUBLE, REAL]") - ), - result.getSchema() - ); - assertEquals( - Sets.newHashSet( - Arrays.asList("John", 25.13), - Arrays.asList("Hank", 30.46), - Arrays.asList("Allen", 15.1) - ), - result.getDataRows() - ); - } - - private ResultSet mockResultSet(Object[]... rows) throws SQLException { - ResultSet resultSet = mock(ResultSet.class); - OngoingStubbing next = when(resultSet.next()); - for (int i = 0; i < rows.length; i++) { - next = next.thenReturn(true); - } - next.thenReturn(false); - - OngoingStubbing getObject = when(resultSet.getObject(anyInt())); - for (Object[] row : rows) { - for (Object val : row) { - getObject = getObject.thenReturn(val); - } - } - return resultSet; - } - - private ResultSetMetaData mockMetaData(Map nameAndTypes, String... aliases) throws SQLException { - ResultSetMetaData metaData = mock(ResultSetMetaData.class); - - OngoingStubbing getColumnName = when(metaData.getColumnName(anyInt())); - for (String name : nameAndTypes.keySet()) { - getColumnName = getColumnName.thenReturn(name); - } - - OngoingStubbing getColumnTypeName = when(metaData.getColumnTypeName(anyInt())); - for (String value : nameAndTypes.values()) { - getColumnTypeName = getColumnTypeName.thenReturn(value); - } - - if (aliases.length > 0) { - OngoingStubbing getColumnLabel = when(metaData.getColumnLabel(anyInt())); - for (String alias : aliases) { - getColumnLabel = getColumnLabel.thenReturn(alias); - } - } - - when(metaData.getColumnCount()).thenReturn(nameAndTypes.size()); - return metaData; - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestConfigTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestConfigTest.java deleted file mode 100644 index a66719e4eb..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestConfigTest.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.correctness.tests; - -import com.amazon.opendistroforelasticsearch.sql.correctness.TestConfig; -import com.google.common.collect.ImmutableMap; -import org.junit.Test; - -import java.util.Map; - -import static java.util.Collections.emptyMap; -import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.emptyString; -import static org.hamcrest.Matchers.hasEntry; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; - -/** - * Tests for {@link TestConfig} - */ -public class TestConfigTest { - - @Test - public void testDefaultConfig() { - TestConfig config = new TestConfig(emptyMap()); - assertThat(config.getESHostUrl(), is(emptyString())); - assertThat( - config.getOtherDbConnectionNameAndUrls(), - allOf( - hasEntry("H2", "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1"), - hasEntry("SQLite", "jdbc:sqlite::memory:") - ) - ); - } - - @Test - public void testCustomESUrls() { - Map args = ImmutableMap.of("esHost", "localhost:9200"); - TestConfig config = new TestConfig(args); - assertThat(config.getESHostUrl(), is("localhost:9200")); - } - - @Test - public void testCustomDbUrls() { - Map args = ImmutableMap.of("otherDbUrls", - "H2=jdbc:h2:mem:test;DB_CLOSE_DELAY=-1," - + "Derby=jdbc:derby:memory:myDb;create=true"); - - TestConfig config = new TestConfig(args); - assertThat( - config.getOtherDbConnectionNameAndUrls(), - allOf( - hasEntry("H2", "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1"), - hasEntry("Derby", "jdbc:derby:memory:myDb;create=true") - ) - ); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestDataSetTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestDataSetTest.java deleted file mode 100644 index e67ba4e8c4..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestDataSetTest.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.correctness.tests; - -import com.amazon.opendistroforelasticsearch.sql.correctness.testset.TestDataSet; -import org.junit.Test; - -import static org.hamcrest.Matchers.contains; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; - -/** - * Tests for {@link TestDataSet} - */ -public class TestDataSetTest { - - @Test - public void testDataSetWithSingleColumnData() { - TestDataSet dataSet = new TestDataSet("test", "mappings", "hello\nworld\n123"); - assertEquals("test", dataSet.getTableName()); - assertEquals("mappings", dataSet.getSchema()); - assertThat( - dataSet.getDataRows(), - contains( - new String[]{"hello"}, - new String[]{"world"}, - new String[]{"123"} - ) - ); - } - - @Test - public void testDataSetWithMultiColumnsData() { - TestDataSet dataSet = new TestDataSet("test", "mappings", "hello,world\n123"); - assertThat( - dataSet.getDataRows(), - contains( - new String[]{"hello", "world"}, - new String[]{"123"} - ) - ); - } - - @Test - public void testDataSetWithEscapedComma() { - TestDataSet dataSet = new TestDataSet("test", "mappings", "hello,\"hello,world,123\"\n123\n\"[abc,def,ghi]\",456"); - assertThat( - dataSet.getDataRows(), - contains( - new String[]{"hello", "hello,world,123"}, - new String[]{"123"}, - new String[]{"[abc,def,ghi]", "456"} - ) - ); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/testset/TestDataSet.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/testset/TestDataSet.java deleted file mode 100644 index 1294afe3c7..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/testset/TestDataSet.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.correctness.testset; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import static com.amazon.opendistroforelasticsearch.sql.utils.StringUtils.unquoteSingleField; -import static java.util.stream.Collectors.joining; - -/** - * Test data set - */ -public class TestDataSet { - - private final String tableName; - private final String schema; - private final List dataRows; - - public TestDataSet(String tableName, String schemaFileContent, String dataFileContent) { - this.tableName = tableName; - this.schema = schemaFileContent; - this.dataRows = splitColumns(dataFileContent, ','); - } - - public String getTableName() { - return tableName; - } - - public String getSchema() { - return schema; - } - - public List getDataRows() { - return dataRows; - } - - /** Split columns in each line by separator and ignore escaped separator(s) in quoted string. */ - private List splitColumns(String content, char separator) { - List result = new ArrayList<>(); - for (String line : content.split("\\r?\\n")) { - - List columns = new ArrayList<>(); - boolean isQuoted = false; - int start = 0; - for (int i = 0; i < line.length(); i++) { - - char c = line.charAt(i); - if (c == separator) { - if (isQuoted) { // Ignore comma(s) in quoted string - continue; - } - - String column = line.substring(start, i); - columns.add(unquoteSingleField(column, "\"")); - start = i + 1; - - } else if (c == '\"') { - isQuoted = !isQuoted; - } - } - - columns.add(unquoteSingleField(line.substring(start), "\"")); - result.add(columns.toArray(new String[0])); - } - return result; - } - - @Override - public String toString() { - int total = dataRows.size(); - return "Test data set :\n" - + " Table name: " + tableName + '\n' - + " Schema: " + schema + '\n' - + " Data rows (first 5 in " + total + "):" - + dataRows.stream(). - limit(5). - map(Arrays::toString). - collect(joining("\n ", "\n ", "\n")); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/admin/MonitoringIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/admin/MonitoringIT.java deleted file mode 100644 index e7e835f0da..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/admin/MonitoringIT.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.doctest.admin; - -import com.amazon.opendistroforelasticsearch.sql.doctest.core.DocTest; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.DocTestConfig; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.Requests; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.response.DataTable; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequest; -import com.amazon.opendistroforelasticsearch.sql.metrics.MetricName; - -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.CURL_REQUEST; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; -import static com.amazon.opendistroforelasticsearch.sql.metrics.MetricName.DEFAULT_CURSOR_REQUEST_COUNT_TOTAL; -import static com.amazon.opendistroforelasticsearch.sql.metrics.MetricName.DEFAULT_CURSOR_REQUEST_TOTAL; -import static com.amazon.opendistroforelasticsearch.sql.metrics.MetricName.FAILED_REQ_COUNT_CB; -import static com.amazon.opendistroforelasticsearch.sql.metrics.MetricName.FAILED_REQ_COUNT_CUS; -import static com.amazon.opendistroforelasticsearch.sql.metrics.MetricName.FAILED_REQ_COUNT_SYS; -import static com.amazon.opendistroforelasticsearch.sql.metrics.MetricName.REQ_COUNT_TOTAL; -import static com.amazon.opendistroforelasticsearch.sql.metrics.MetricName.REQ_TOTAL; -import static com.amazon.opendistroforelasticsearch.sql.plugin.RestSqlStatsAction.STATS_API_ENDPOINT; - -/** - * Doc test for plugin monitoring functionality - */ -@DocTestConfig(template = "admin/monitoring.rst") -public class MonitoringIT extends DocTest { - - @Section - public void nodeStats() { - section( - title("Node Stats"), - description("The meaning of fields in the response is as follows:\n\n" + fieldDescriptions()), - example( - description(), - getStats(), - queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ) - ); - } - - private String fieldDescriptions() { - DataTable table = new DataTable(new String[]{ "Field name", "Description" }); - table.addRow(row(REQ_TOTAL, "Total count of request")); - table.addRow(row(REQ_COUNT_TOTAL, "Total count of request within the interval")); - table.addRow(row(DEFAULT_CURSOR_REQUEST_TOTAL, "Total count of simple cursor request")); - table.addRow(row(DEFAULT_CURSOR_REQUEST_COUNT_TOTAL, "Total count of simple cursor request within the interval")); - table.addRow(row(FAILED_REQ_COUNT_SYS, "Count of failed request due to system error within the interval")); - table.addRow(row(FAILED_REQ_COUNT_CUS, "Count of failed request due to bad request within the interval")); - table.addRow(row(FAILED_REQ_COUNT_CB, "Indicate if plugin is being circuit broken within the interval")); - - return table.toString(); - } - - private String[] row(MetricName name, String description) { - return new String[] { name.getName(), description }; - } - - private Requests getStats() { - return new Requests(restClient(), new SqlRequest("GET", STATS_API_ENDPOINT, "")); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/admin/PluginSettingIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/admin/PluginSettingIT.java deleted file mode 100644 index af348310d0..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/admin/PluginSettingIT.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.doctest.admin; - -import com.amazon.opendistroforelasticsearch.sql.doctest.core.DocTest; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.DocTestConfig; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.Example; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.ListItems; -import com.amazon.opendistroforelasticsearch.sql.executor.Format; -import com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; -import org.elasticsearch.common.settings.Setting; - -import java.util.Arrays; -import java.util.EnumSet; -import java.util.stream.Collectors; - -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.CURL_REQUEST; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.CURSOR_ENABLED; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.CURSOR_FETCH_SIZE; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.CURSOR_KEEPALIVE; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.QUERY_ANALYSIS_ENABLED; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_SUGGESTION; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_THRESHOLD; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.QUERY_RESPONSE_FORMAT; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.QUERY_SLOWLOG; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.SQL_ENABLED; -import static org.elasticsearch.common.settings.Setting.Property; -import static org.elasticsearch.common.settings.Setting.Property.Dynamic; -import static org.elasticsearch.common.settings.Setting.Property.Final; -import static org.elasticsearch.common.settings.Setting.Property.IndexScope; -import static org.elasticsearch.common.settings.Setting.Property.NodeScope; -import static org.elasticsearch.common.settings.Settings.EMPTY; - -/** - * Doc test for plugin settings. - */ -@DocTestConfig(template = "admin/settings.rst", testData = {"accounts.json"}) -public class PluginSettingIT extends DocTest { - - private static final SqlSettings SETTINGS = new SqlSettings(); - - @Section(1) - public void sqlEnabledSetting() { - docSetting( - SQL_ENABLED, - "You can disable SQL plugin to reject all coming requests.", - false, - "SELECT * FROM accounts" - ); - } - - @Section(2) - public void slowLogSetting() { - docSetting( - QUERY_SLOWLOG, - "You can configure the time limit (seconds) for slow query which would be logged as " + - "'Slow query: elapsed=xxx (ms)' in elasticsearch.log.", - 10 - ); - } - - @Section(3) - public void queryAnalysisEnabledSetting() { - docSetting( - QUERY_ANALYSIS_ENABLED, - "You can disable query analyzer to bypass strict syntactic and semantic analysis.", - false - ); - } - - @Section(4) - public void semanticSuggestionSetting() { - docSetting( - QUERY_ANALYSIS_SEMANTIC_SUGGESTION, - "You can enable query analyzer to suggest correct field names for quick fix.", - true, - "SELECT first FROM accounts" - ); - } - - @Section(5) - public void semanticAnalysisThresholdSetting() { - docSetting( - QUERY_ANALYSIS_SEMANTIC_THRESHOLD, - "Because query analysis needs to build semantic context in memory, index with large number of field " + - "would be skipped. You can update it to apply analysis to smaller or larger index as needed.", - 50 - ); - } - - @Section(6) - public void responseFormatSetting() { - docSetting( - QUERY_RESPONSE_FORMAT, - String.format("User can set default response format of the query. " + - "The supported format includes: %s.", Arrays.stream(Format.values()) - .map(Format::getFormatName) - .collect(Collectors.joining(","))), - Format.JSON.getFormatName(), - "SELECT firstname, lastname, age FROM accounts ORDER BY age LIMIT 2" - ); - } - - @Section(7) - public void cursorEnabledSetting() { - docSetting( - CURSOR_ENABLED, - "User can enable/disable pagination for all queries that are supported.", - true - ); - } - - @Section(8) - public void cursorDefaultFetchSizeSetting() { - docSetting( - CURSOR_FETCH_SIZE, - "User can set the default fetch_size for all queries that are supported by pagination. " + - "Explicit `fetch_size` passed in request will override this value", - 50 - ); - } - - @Section(9) - public void cursorDefaultContextKeepAliveSetting() { - docSetting( - CURSOR_KEEPALIVE, - "User can set this value to indicate how long the cursor context should be kept open. " + - "Cursor contexts are resource heavy, and a lower value should be used if possible.", - "5m" - ); - } - - /** - * Generate content for sample queries with setting changed to new value. - * Finally setting will be reverted to avoid potential impact on other test cases. - */ - private void docSetting(String name, String description, Object sampleValue, String... sampleQueries) { - try { - section( - title(name), - description(description + "\n\n" + listSettingDetails(name)), - createExamplesForSettingChangeQueryAndOtherSampleQueries(name, sampleValue, sampleQueries) - ); - } finally { - // Make sure the change is removed - try { - put(name, null).queryResponse(); - } catch (Exception e) { - // Ignore - } - } - } - - private Example[] createExamplesForSettingChangeQueryAndOtherSampleQueries(String name, - Object sampleValue, - String[] sampleQueries) { - Example[] examples = new Example[sampleQueries.length + 1]; - examples[0] = example("You can update the setting with a new value like this.", - put(name, sampleValue), - queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE)); - - for (int i = 0; i < sampleQueries.length; i++) { - examples[i + 1] = example("Query result after the setting updated is like:", - post(sampleQueries[i]), - queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE)); - } - return examples; - } - - private String listSettingDetails(String name) { - Setting setting = SETTINGS.getSetting(name); - ListItems list = new ListItems(); - - list.addItem(StringUtils.format("The default value is %s.", setting.getDefault(EMPTY))); - - EnumSet properties = setting.getProperties(); - if (properties.contains(NodeScope)) { - list.addItem("This setting is node scope."); - } else if (properties.contains(IndexScope)) { - list.addItem("This setting is index scope."); - } - - if (properties.contains(Dynamic)) { - list.addItem("This setting can be updated dynamically."); - } else if (properties.contains(Final)) { - list.addItem("This setting is not updatable."); - } - return list.toString(); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/beyond/FullTextIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/beyond/FullTextIT.java deleted file mode 100644 index 190dc2223d..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/beyond/FullTextIT.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.doctest.beyond; - -import com.amazon.opendistroforelasticsearch.sql.doctest.core.DocTest; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.DocTestConfig; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; - -@DocTestConfig(template = "beyond/fulltext.rst", testData = {"accounts.json"}) -public class FullTextIT extends DocTest { - - @Section(1) - public void matchQuery() { - section( - title("Match Query"), - description( - "Match query is the standard query for full-text search in Elasticsearch. Both ``MATCHQUERY`` and", - "``MATCH_QUERY`` are functions for performing match query." - ), - example( - description("Both functions can accept field name as first argument and a text as second argument."), - post(multiLine( - "SELECT account_number, address", - "FROM accounts", - "WHERE MATCH_QUERY(address, 'Holmes')" - )) - ), - example( - description("Both functions can also accept single argument and be used in the following manner."), - post(multiLine( - "SELECT account_number, address", - "FROM accounts", - "WHERE address = MATCH_QUERY('Holmes')" - )) - ) - ); - } - - @Section(2) - public void multiMatchQuery() { - section( - title("Multi-match Query"), - description( - "Besides match query against a single field, you can search for a text with multiple fields.", - "Function ``MULTI_MATCH``, ``MULTIMATCH`` and ``MULTIMATCHQUERY`` are provided for this." - ), - example( - description( - "Each preceding function accepts ``query`` for a text and ``fields`` for field names or pattern", - "that the text given is searched against. For example, the following query is searching for", - "documents in index accounts with 'Dale' as either firstname or lastname." - ), - post(multiLine( - "SELECT firstname, lastname", - "FROM accounts", - "WHERE MULTI_MATCH('query'='Dale', 'fields'='*name')" - )) - ) - ); - } - - @Section(3) - public void queryStringQuery() { - section( - title("Query String Query"), - description( - "Query string query parses and splits a query string provided based on Lucene query string syntax.", - "The mini language supports logical connectives, wildcard, regex and proximity search. Please refer", - "to official documentation for more details. Note that an error is thrown in the case of any invalid", - "syntax in query string." - ), - example( - description( - "``QUERY`` function accepts query string and returns true or false respectively for document", - "that matches the query string or not." - ), - post(multiLine( - "SELECT account_number, address", - "FROM accounts", - "WHERE QUERY('address:Lane OR address:Street')" - )) - ) - ); - } - - @Section(4) - public void matchPhraseQuery() { - section( - title("Match Phrase Query"), - description( - "Match phrase query is similar to match query but it is used for matching exact phrases.", - "``MATCHPHRASE``, ``MATCH_PHRASE`` and ``MATCHPHRASEQUERY`` are provided for this purpose." - ), - example( - description(), - post(multiLine( - "SELECT account_number, address", - "FROM accounts", - "WHERE MATCH_PHRASE(address, '880 Holmes Lane')" - )) - ) - ); - } - - @Section(5) - public void scoreQuery() { - section( - title("Score Query"), - description( - "Elasticsearch supports to wrap a filter query so as to return a relevance score along with", - "every matching document. ``SCORE``, ``SCOREQUERY`` and ``SCORE_QUERY`` can be used for this." - ), - example( - description( - "The first argument is a match query expression and the second argument is for an optional", - "floating point number to boost the score. The default value is 1.0. Apart from this, an", - "implicit variable ``_score`` is available so you can return score for each document or", - "use it for sorting." - ), - post(multiLine( - "SELECT account_number, address, _score", - "FROM accounts", - "WHERE SCORE(MATCH_QUERY(address, 'Lane'), 0.5) OR", - " SCORE(MATCH_QUERY(address, 'Street'), 100)", - "ORDER BY _score" - )) - ) - ); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/beyond/PartiQLIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/beyond/PartiQLIT.java deleted file mode 100644 index 969cbbcfe5..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/beyond/PartiQLIT.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.doctest.beyond; - -import com.amazon.opendistroforelasticsearch.sql.doctest.core.DocTest; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.DocTestConfig; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.Example; -import com.amazon.opendistroforelasticsearch.sql.utils.JsonPrettyFormatter; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.TestData.TEST_DATA_FOLDER_ROOT; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getResourceFilePath; - -@DocTestConfig(template = "beyond/partiql.rst", testData = {"employees_nested.json"}) -public class PartiQLIT extends DocTest { - - @Section(1) - public void showTestData() { - section( - title("Test Data"), - description( - "The test index ``employees_nested`` used by all examples in this document is very similar to", - "the one used in official PartiQL documentation." - ), - createDummyExampleForTestData("employees_nested.json") - ); - } - - @Section(2) - public void queryNestedCollection() { - section( - title("Querying Nested Collection"), - description( - "In SQL-92, a database table can only have tuples that consists of scalar values.", - "PartiQL extends SQL-92 to allow you query and unnest nested collection conveniently.", - "In Elasticsearch world, this is very useful for index with object or nested field." - ), - example( - title("Unnesting a Nested Collection"), - description( - "In the following example, it finds nested document (project) with field value (name)", - "that satisfies the predicate (contains 'security'). Note that because each parent document", - "can have more than one nested documents, the matched nested document is flattened. In other", - "word, the final result is the Cartesian Product between parent and nested documents." - ), - post(multiLine( - "SELECT e.name AS employeeName,", - " p.name AS projectName", - "FROM employees_nested AS e,", - " e.projects AS p", - "WHERE p.name LIKE '%security%'" - )) - ), - /* - Issue: https://github.com/opendistro-for-elasticsearch/sql/issues/397 - example( - title("Preserving Parent Information with LEFT JOIN"), - description( - "The query in the preceding example is very similar to traditional join queries, except ``ON`` clause missing.", - "This is because it is implicitly in the nesting of nested documents (projects) into parent (employee). Therefore,", - "you can use ``LEFT JOIN`` to preserve the information in parent document associated." - ), - post( - "SELECT e.id AS id, " + - " e.name AS employeeName, " + - " e.title AS title, " + - " p.name AS projectName " + - "FROM employees_nested AS e " + - "LEFT JOIN e.projects AS p" - ) - )*/ - example( - title("Unnesting in Existential Subquery"), - description( - "Alternatively, a nested collection can be unnested in subquery to check if it", - "satisfies a condition." - ), - post(multiLine( - "SELECT e.name AS employeeName", - "FROM employees_nested AS e", - "WHERE EXISTS (", - " SELECT *", - " FROM e.projects AS p", - " WHERE p.name LIKE '%security%'", - ")" - )) - )/*, - Issue: https://github.com/opendistro-for-elasticsearch/sql/issues/398 - example( - title("Aggregating over a Nested Collection"), - description( - "After unnested, a nested collection can be aggregated just like a regular field." - ), - post(multiLine( - "SELECT", - " e.name AS employeeName,", - " COUNT(p) AS cnt", - "FROM employees_nested AS e,", - " e.projects AS p", - "WHERE p.name LIKE '%security%'", - "GROUP BY e.id, e.name", - "HAVING COUNT(p) >= 1" - ) - )) - */ - ); - } - - private Example createDummyExampleForTestData(String fileName) { - Example example = new Example(); - example.setTitle("Employees"); - example.setDescription(""); - example.setResult(parseJsonFromTestData(fileName)); - return example; - } - - /** Concat and pretty format document at odd number line in bulk request file */ - private String parseJsonFromTestData(String fileName) { - Path path = Paths.get(getResourceFilePath(TEST_DATA_FOLDER_ROOT + fileName)); - try { - List lines = Files.readAllLines(path); - String json = IntStream.range(0, lines.size()). - filter(i -> i % 2 == 1). - mapToObj(lines::get). - collect(Collectors.joining(",","{\"employees\":[", "]}")); - return JsonPrettyFormatter.format(json); - } catch (IOException e) { - throw new IllegalStateException("Failed to load test data: " + path, e); - } - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/DocTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/DocTest.java deleted file mode 100644 index 62b711ff2a..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/DocTest.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.doctest.core; - -import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.DocTestConfig; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.DocBuilder; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.markup.Document; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.markup.RstDocument; -import com.amazon.opendistroforelasticsearch.sql.esintgtest.CustomExternalTestCluster; -import com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils; -import com.carrotsearch.randomizedtesting.AnnotatedMethodProvider; -import com.carrotsearch.randomizedtesting.TestMethodAndParams; -import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering; -import com.carrotsearch.randomizedtesting.annotations.TestMethodProviders; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.elasticsearch.test.TestCluster; - -import java.io.IOException; -import java.io.PrintWriter; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Comparator; - -import static com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope; -import static java.nio.file.StandardOpenOption.APPEND; -import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; - -/** - * Documentation test base class - */ -@TestMethodProviders({DocTest.SectionMethod.class}) -@TestCaseOrdering(DocTest.SectionOrder.class) -@ESIntegTestCase.SuiteScopeTestCase -@ClusterScope(scope= SUITE, numDataNodes=1, supportsDedicatedMasters=false, transportClientRatio=1) -@ThreadLeakScope(Scope.NONE) -public abstract class DocTest extends ESIntegTestCase implements DocBuilder { - - @Override - protected void setupSuiteScopeCluster() { - DocTestConfig config = getClass().getAnnotation(DocTestConfig.class); - loadTestData(config); - copyTemplateToDocument(config); - } - - @Override - public RestClient restClient() { - return getRestClient(); - } - - @Override - public Document openDocument() { - DocTestConfig config = getClass().getAnnotation(DocTestConfig.class); - Path docPath = absolutePath(config.template()); - try { - PrintWriter docWriter = new PrintWriter(Files.newBufferedWriter(docPath, APPEND)); - return new RstDocument(docWriter); - } catch (IOException e) { - throw new IllegalStateException("Failed to open document file " + docPath, e); - } - } - - private void loadTestData(DocTestConfig config) { - String[] testFilePaths = config.testData(); - TestData testData = new TestData(testFilePaths); - testData.loadToES(this); - } - - private void copyTemplateToDocument(DocTestConfig config) { - Path docPath = absolutePath(config.template()); - Template template = new Template(config.template()); - template.copyToDocument(docPath); - } - - /** - * Method annotated by {@link Section} will be treated as test method. - */ - public static class SectionMethod extends AnnotatedMethodProvider { - public SectionMethod() { - super(Section.class); - } - } - - /** - * Test methods will execute in order defined by value in {@link Section} annotation. - */ - public static class SectionOrder implements Comparator { - @Override - public int compare(TestMethodAndParams method1, TestMethodAndParams method2) { - return Integer.compare(order(method1), order(method2)); - } - - private int order(TestMethodAndParams method) { - Section section = method.getTestMethod().getAnnotation(Section.class); - return section.value(); - } - } - - private Path absolutePath(String templateRelativePath) { - return Paths.get(TestUtils.getResourceFilePath(DOCUMENT_FOLDER_ROOT + templateRelativePath)); - } - - @Override - protected TestCluster buildTestCluster(Scope scope, long seed) throws IOException { - - String clusterAddresses = System.getProperty(TESTS_CLUSTER); - - if (Strings.hasLength(clusterAddresses)) { - String[] stringAddresses = clusterAddresses.split(","); - TransportAddress[] transportAddresses = new TransportAddress[stringAddresses.length]; - int i = 0; - for (String stringAddress : stringAddresses) { - URL url = new URL("http://" + stringAddress); - InetAddress inetAddress = InetAddress.getByName(url.getHost()); - transportAddresses[i++] = new TransportAddress(new InetSocketAddress(inetAddress, url.getPort())); - } - return new CustomExternalTestCluster(createTempDir(), externalClusterClientSettings(), - transportClientPlugins(), transportAddresses); - } - return super.buildTestCluster(scope, seed); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/Template.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/Template.java deleted file mode 100644 index 61738a8173..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/Template.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.doctest.core; - -import com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; - -import static java.nio.file.StandardCopyOption.COPY_ATTRIBUTES; -import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; - -/** - * Abstraction for document template file - */ -public class Template { - - private static final String TEMPLATE_FOLDER_ROOT = "src/test/resources/doctest/templates/"; - - private final Path templateFullPath; - - public Template(String templateRelativePath) { - this.templateFullPath = Paths.get(TestUtils.getResourceFilePath(TEMPLATE_FOLDER_ROOT + templateRelativePath)); - } - - /** - * Copy template file to target document. Replace it if existing. - * @param docFullPath full path of target document - */ - public void copyToDocument(Path docFullPath) { - try { - Files.createDirectories(docFullPath.getParent()); - Files.copy(templateFullPath, docFullPath, REPLACE_EXISTING, COPY_ATTRIBUTES); - } catch (IOException e) { - throw new IllegalStateException(StringUtils.format( - "Failed to copy from template [%s] to document file [%s]", templateFullPath, docFullPath), e); - } - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/TestData.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/TestData.java deleted file mode 100644 index 4b2e1788c4..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/TestData.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.doctest.core; - -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; - -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.createIndexByRestClient; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getResourceFilePath; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.loadDataByRestClient; - -/** - * Test data for document generation - */ -public class TestData { - - public static final String MAPPINGS_FOLDER_ROOT = "src/test/resources/doctest/mappings/"; - public static final String TEST_DATA_FOLDER_ROOT = "src/test/resources/doctest/testdata/"; - - private final String[] testFilePaths; - - public TestData(String[] testFilePaths) { - this.testFilePaths = testFilePaths; - } - - /** - * Load test data in file to Elaticsearch cluster via client. - * @param test current test instance - */ - public void loadToES(DocTest test) { - for (String filePath : testFilePaths) { - String indexName = indexName(filePath); - try { - createIndexByRestClient(test.restClient(), indexName, getIndexMapping(filePath)); - loadDataByRestClient(test.restClient(), indexName, TEST_DATA_FOLDER_ROOT + filePath); - } catch (Exception e) { - throw new IllegalStateException(StringUtils.format( - "Failed to load mapping and test filePath from %s", filePath), e); - } - test.ensureGreen(indexName); - } - } - - /** - * Use file name (without file extension) as index name implicitly. - * For example, for 'testdata/accounts.json', 'accounts' will be used. - */ - private String indexName(String filePath) { - return filePath.substring( - filePath.lastIndexOf(File.separatorChar) + 1, - filePath.lastIndexOf('.') - ); - } - - private String getIndexMapping(String filePath) throws IOException { - Path path = Paths.get(getResourceFilePath(MAPPINGS_FOLDER_ROOT + filePath)); - if (Files.notExists(path)) { - return ""; - } - return new String(Files.readAllBytes(path)); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/DocBuilder.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/DocBuilder.java deleted file mode 100644 index 6f7c091e76..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/DocBuilder.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.doctest.core.builder; - -import com.amazon.opendistroforelasticsearch.sql.doctest.core.markup.Document; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequest; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; -import com.google.common.base.Strings; -import org.elasticsearch.client.RestClient; - -import java.util.Arrays; - -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequest.UrlParam; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.KIBANA_REQUEST; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.TABLE_RESPONSE; -import static com.amazon.opendistroforelasticsearch.sql.plugin.RestSqlAction.EXPLAIN_API_ENDPOINT; -import static com.amazon.opendistroforelasticsearch.sql.plugin.RestSqlAction.QUERY_API_ENDPOINT; - -/** - * Build document by custom DSL. To make it more readable, each doc test needs to implement this interface - * and provide things required, such as client connection and document handle. As benefit, they can use the - * DSL to build document in readable and fluent way. - */ -public interface DocBuilder { - - String DOCUMENT_FOLDER_ROOT = "/docs/user/"; - String IMAGE_FOLDER_PATH = DOCUMENT_FOLDER_ROOT + "img/"; - - /** - * Get client connection to cluster for sending request - * @return REST client - */ - RestClient restClient(); - - /** - * Open document file to write - * @return document class - */ - Document openDocument(); - - default void section(String title, String description, Example... examples) { - section(title, description, new String[0], examples); - } - - /** - * Entry point to start building document by DSL. - * Each section consists of: - * 1. Title - * 2. Description - * 3. Example(s) - * 3.1 Description - * 3.2 [Sample request] - * 3.3 [Explain request] - * 3.4 [Explain output] - * 3.5 [Result set] - * - * @param title title of the section - * @param description description paragraph - * @param examples examples for the section - */ - default void section(String title, String description, String[] images, Example... examples) { - try (Document document = openDocument()) { - document.section(title); - - if (!description.isEmpty()) { - document.subSection("Description").paragraph(description); - } - - if (images.length > 0) { - document.subSection("Syntax"); - for (String image : images) { - // Convert image name ex. "rdd/queryStatement.png" to "queryStatement" as description. - String imageDesc = image.substring(image.lastIndexOf('/') + 1, image.lastIndexOf('.')); - document.image("Rule ``" + imageDesc + "``", IMAGE_FOLDER_PATH + image); - } - } - - for (int i = 0; i < examples.length; i++) { - String exampleTitle; - if (examples.length > 1) { - exampleTitle = "Example " + (i + 1); - } else { - exampleTitle = "Example"; - } - if (!Strings.isNullOrEmpty(examples[i].getTitle())) { - exampleTitle += (": " + examples[i].getTitle()); - } - document.subSection(exampleTitle); - - Example example = examples[i]; - if (!example.getDescription().isEmpty()) { - document.paragraph(example.getDescription()); - } - - document.codeBlock("SQL query", example.getQuery()). - codeBlock("Explain query", example.getExplainQuery()). - codeBlock("Explain", example.getExplainResult()); - - if (example.isTable()) { - document.table("Result set", example.getResult()); - } else { - document.codeBlock("Result set", example.getResult()); - } - } - } - } - - default Example example(String description, Requests requests) { - return example("", description, requests); - } - - /** Construct an example by default query and explain format */ - default Example example(String title, String description, Requests requests) { - return example(title, description, requests, - queryFormat(KIBANA_REQUEST, TABLE_RESPONSE), - explainFormat(IGNORE_REQUEST, PRETTY_JSON_RESPONSE) - ); - } - - default Example example(String description, - Requests requests, - Formats queryFormat, - Formats explainFormat) { - return example("", description, requests, queryFormat, explainFormat); - } - - default Example example(String title, - String description, - Requests requests, - Formats queryFormat, - Formats explainFormat) { - Example example = new Example(); - example.setTitle(title); - example.setDescription(description); - example.setQuery(queryFormat.format(requests.query())); - example.setTable(queryFormat.isTableFormat()); - example.setResult(queryFormat.format(requests.queryResponse())); - example.setExplainQuery(explainFormat.format(requests.explain())); - example.setExplainResult(explainFormat.format(requests.explainResponse())); - return example; - } - - /** Simple method just for readability */ - default String title(String title) { - return title; - } - - default String description(String... sentences) { - return String.join(" ", sentences); - } - - default String[] images(String... images) { - return images; - } - - default Formats queryFormat(SqlRequestFormat requestFormat, SqlResponseFormat responseFormat) { - return new Formats(requestFormat, responseFormat); - } - - default Formats explainFormat(SqlRequestFormat requestFormat, SqlResponseFormat responseFormat) { - return new Formats(requestFormat, responseFormat); - } - - default Requests get(String sql) { - Body body = new Body("\"query\":\"" + sql + "\""); - return new Requests( - restClient(), - new SqlRequest("GET", QUERY_API_ENDPOINT, "", new UrlParam("sql", sql)), - new SqlRequest("POST", EXPLAIN_API_ENDPOINT, body.toString()) - ); - } - - default Requests put(String name, Object value) { - String setting = value == null ? - StringUtils.format("\"%s\": {\"%s\": null}", "transient", name) : - StringUtils.format("\"%s\": {\"%s\": \"%s\"}", "transient", name, value); - return new Requests( - restClient(), - new SqlRequest("PUT", "/_opendistro/_sql/settings", new Body(setting).toString()), - null - ); - } - - default String multiLine(String... lines) { - return String.join("\\n", lines); - } - - /** Query by a simple SQL is too common and deserve a dedicated overload method */ - default Requests post(String sql, UrlParam... params) { - return post(new Body("\"query\":\"" + sql + "\""), params); - } - - default Requests post(Body body, UrlParam... params) { - String bodyStr = body.toString(); - return new Requests( - restClient(), - new SqlRequest("POST", QUERY_API_ENDPOINT, bodyStr, params), - new SqlRequest("POST", EXPLAIN_API_ENDPOINT, bodyStr) - ); - } - - default Body body(String... fieldValues) { - return new Body(fieldValues); - } - - default UrlParam[] params(String... keyValues) { - return Arrays.stream(keyValues).map(UrlParam::new).toArray(UrlParam[]::new); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/Example.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/Example.java deleted file mode 100644 index f068a78016..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/Example.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.doctest.core.builder; - -/** - * Example value object. - */ -public class Example { - - /** Title for the example section */ - private String title; - - /** Description for the example */ - private String description; - - /** Sample SQL query */ - private String query; - - /** Query result set */ - private String result; - - /** Is result set formatted in table (markup handle table in different way */ - private boolean isTable; - - /** Explain query correspondent to the sample query */ - private String explainQuery; - - /** Result of explain */ - private String explainResult; - - public String getTitle() { - return title; - } - - public String getDescription() { - return description; - } - - public String getQuery() { - return query; - } - - public String getResult() { - return result; - } - - public boolean isTable() { - return isTable; - } - - public String getExplainQuery() { - return explainQuery; - } - - public String getExplainResult() { - return explainResult; - } - - public void setTitle(String title) { - this.title = title; - } - - public void setDescription(String description) { - this.description = description; - } - - public void setQuery(String query) { - this.query = query; - } - - public void setResult(String result) { - this.result = result; - } - - public void setTable(boolean table) { - isTable = table; - } - - public void setExplainQuery(String explainQuery) { - this.explainQuery = explainQuery; - } - - public void setExplainResult(String explainResult) { - this.explainResult = explainResult; - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/markup/RstDocument.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/markup/RstDocument.java deleted file mode 100644 index 52ec23a98e..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/markup/RstDocument.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.doctest.core.markup; - -import com.google.common.base.Strings; - -import java.io.PrintWriter; - -/** - * ReStructure Text document - */ -public class RstDocument implements Document { - - private final PrintWriter docWriter; - - public RstDocument(PrintWriter docWriter) { - this.docWriter = docWriter; - } - - @Override - public Document section(String title) { - return printTitleWithUnderline(title, "="); - } - - @Override - public Document subSection(String title) { - return printTitleWithUnderline(title, "-"); - } - - @Override - public Document paragraph(String text) { - return println(text); - } - - @Override - public Document codeBlock(String description, String code) { - if (!Strings.isNullOrEmpty(code)) { - return println(description + "::", indent(code)); - } - return this; - } - - @Override - public Document table(String description, String table) { - if (!Strings.isNullOrEmpty(table)) { - // RST table is different and not supposed to indent - return println(description + ":", table); - } - return this; - } - - @Override - public Document image(String description, String filePath) { - return println( - description + ":", - ".. image:: " + filePath - ); - } - - @Override - public void close() { - docWriter.close(); - } - - private Document printTitleWithUnderline(String title, String underlineChar) { - return print( - title, - Strings.repeat(underlineChar, title.length()) - ); - } - - /** Print each line with a blank line at last */ - private Document print(String... lines) { - for (String line : lines) { - docWriter.println(line); - } - docWriter.println(); - return this; - } - - /** Print each line with a blank line followed */ - private Document println(String... lines) { - for (String line : lines) { - docWriter.println(line); - docWriter.println(); - } - return this; - } - - private String indent(String text) { - return "\t" + text.replaceAll("\\n", "\n\t"); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/request/SqlRequest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/request/SqlRequest.java deleted file mode 100644 index 4d8986fbed..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/request/SqlRequest.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.doctest.core.request; - -import com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponse; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.ResponseException; -import org.elasticsearch.client.RestClient; - -import java.io.IOException; - -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.KIBANA_REQUEST; - -/** - * Request to SQL plugin to isolate Elasticsearch native request - */ -public class SqlRequest { - - public static final SqlRequest NONE = null; - - /** Native Elasticsearch request object */ - private final Request request; - - public SqlRequest(String method, String endpoint, String body, UrlParam... params) { - this.request = makeRequest(method, endpoint, body, params); - } - - /** - * Send request to Elasticsearch via client and create response for it. - * @param client restful client connection - * @return sql response - */ - public SqlResponse send(RestClient client) { - try { - return new SqlResponse(client.performRequest(request)); - } catch (IOException e) { - // Some test may expect failure - if (e instanceof ResponseException) { - return new SqlResponse(((ResponseException) e).getResponse()); - } - - throw new IllegalStateException(StringUtils.format( - "Exception occurred during sending request %s", KIBANA_REQUEST.format(this)), e); - } - } - - /** - * Expose request for request formatter. - * @return native Elasticsearch format - */ - public Request request() { - return request; - } - - private Request makeRequest(String method, String endpoint, String body, UrlParam[] params) { - Request request = new Request(method, endpoint); - request.setJsonEntity(body); - for (UrlParam param : params) { - request.addParameter(param.key, param.value); - } - - RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); - restOptionsBuilder.addHeader("Content-Type", "application/json"); - request.setOptions(restOptionsBuilder); - return request; - } - - public static class UrlParam { - private String key; - private String value; - - public UrlParam(String key, String value) { - this.key = key; - this.value = value; - } - - public UrlParam(String keyValue) { - int equality = keyValue.indexOf('='); - if (equality == -1) { - throw new IllegalArgumentException(String.format( - "Key value pair is in bad format [%s]", keyValue)); - } - - this.key = keyValue.substring(0, equality); - this.value = keyValue.substring(equality + 1); - } - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/request/SqlRequestFormat.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/request/SqlRequestFormat.java deleted file mode 100644 index c6e32d9f8c..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/request/SqlRequestFormat.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.doctest.core.request; - -import com.amazon.opendistroforelasticsearch.sql.utils.JsonPrettyFormatter; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; -import com.google.common.base.Charsets; -import com.google.common.io.CharStreams; -import org.apache.http.Header; -import org.elasticsearch.client.Request; -import org.json.JSONObject; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import static java.util.stream.Collectors.joining; - -/** - * Different SQL request formats. - */ -public enum SqlRequestFormat { - - IGNORE_REQUEST { - @Override - public String format(SqlRequest request) { - return ""; - } - }, - CURL_REQUEST { - @Override - public String format(SqlRequest sqlRequest) { - Request request = sqlRequest.request(); - StringBuilder str = new StringBuilder(); - str.append(">> curl "); - - List
headers = request.getOptions().getHeaders(); - if (!headers.isEmpty()) { - str.append(headers.stream(). - map(header -> StringUtils.format("-H '%s: %s'", header.getName(), header.getValue())). - collect(joining(" ", "", " "))); - } - - str.append(StringUtils.format("-X %s ", request.getMethod())). - append("localhost:9200").append(request.getEndpoint()); - - if (!request.getParameters().isEmpty()) { - str.append(formatParams(request.getParameters())); - } - - String body = body(request); - if (!body.isEmpty()) { - str.append(" -d '"). - append(body). - append('\''); - } - return str.toString(); - } - }, - KIBANA_REQUEST { - @Override - public String format(SqlRequest sqlRequest) { - Request request = sqlRequest.request(); - StringBuilder str = new StringBuilder(); - str.append(request.getMethod()). - append(" "). - append(request.getEndpoint()); - - if (!request.getParameters().isEmpty()) { - str.append(formatParams(request.getParameters())); - } - - str.append('\n'). - append(body(request)); - return str.toString(); - } - }; - - /** - * Format SQL request to specific format for documentation. - * @param request sql request - * @return string in specific format - */ - public abstract String format(SqlRequest request); - - @SuppressWarnings("UnstableApiUsage") - protected String body(Request request) { - String body = ""; - try { - InputStream content = request.getEntity().getContent(); - String rawBody = CharStreams.toString(new InputStreamReader(content, Charsets.UTF_8)); - if (!rawBody.isEmpty()) { - JSONObject json = new JSONObject(rawBody); - String sql = json.optString("query"); // '\\n' in literal is replaced by '\n' after unquote - body = JsonPrettyFormatter.format(rawBody); - - // Format and replace multi-line sql literal - if (!sql.isEmpty() && sql.contains("\n")) { - String multiLineSql = Arrays.stream(sql.split("\\n")). // '\\n' is to escape backslash in regex - collect(joining("\n\t", - "\"\"\"\n\t", - "\n\t\"\"\"")); - body = body.replace("\"" + sql.replace("\n", "\\n") + "\"", multiLineSql); - } - } - } catch (IOException e) { - throw new IllegalStateException("Failed to parse and format body from request", e); - } - return body; - } - - protected String formatParams(Map params) { - return params.entrySet().stream(). - map(e -> e.getKey() + "=" + e.getValue()). - collect(joining("&", "?", "")); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/response/DataTable.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/response/DataTable.java deleted file mode 100644 index 9e38a799e5..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/response/DataTable.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.doctest.core.response; - -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; -import com.google.common.base.Strings; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -/** - * Data table that represent rows of data with a header. - * For now the format is actually in ReST and may need to decouple later. - */ -public class DataTable { - - private final int[] maxWidths; - private final Object[] header; - private final List rows; - - public DataTable(Object[] header) { - this.maxWidths = new int[header.length]; - this.header = header; - this.rows = new ArrayList<>(); - updateMaxWidthForEachColumn(header); - } - - public void addRow(Object[] row) { - rows.add(row); - updateMaxWidthForEachColumn(row); - } - - @Override - public String toString() { - StringBuilder str = new StringBuilder(); - String format = format(); - String separateLine1 = separateLine("-"); - String separateLine2 = separateLine("="); - - str.append(separateLine1). - append('\n'). - append(StringUtils.format(format, header)). - append('\n'). - append(separateLine2). - append('\n'); - - for (Object[] row : rows) { - str.append(StringUtils.format(format, row)). - append('\n'). - append(separateLine("-")). - append('\n'); - } - return str.toString(); - } - - private void updateMaxWidthForEachColumn(Object[] row) { - for (int i = 0; i < row.length; i++) { - maxWidths[i] = Math.max(maxWidths[i], String.valueOf(row[i]).length()); - } - } - - private String separateLine(String separator) { - return Arrays.stream(maxWidths). - mapToObj(width -> Strings.repeat(separator, width)). - collect(Collectors.joining("+", "+", "+")); - } - - /** - * Format as Java String.format needs to make use of auto pad feature. - * For example, to ensure width of 10 and pad spaces, we need to String.format("%10s", str); - */ - private String format() { - return Arrays.stream(maxWidths). - mapToObj(width -> "%" + width + "s"). - collect(Collectors.joining("|", "|", "|")); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/response/SqlResponse.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/response/SqlResponse.java deleted file mode 100644 index a50036475e..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/response/SqlResponse.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.doctest.core.response; - -import com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils; -import org.elasticsearch.client.Response; -import org.json.JSONException; -import org.json.JSONObject; - -import java.io.IOException; - -/** - * Response from SQL plugin - */ -public class SqlResponse { - - public static final SqlResponse NONE = null; - - /** Native Elasticsearch response */ - private final Response response; - - public SqlResponse(Response response) { - this.response = response; - } - - /** - * Parse body in the response. - * @return response body - */ - public String body() { - try { - return replaceChangingFields(TestUtils.getResponseBody(response, true)); - } catch (IOException e) { - throw new IllegalStateException("Failed to read response body", e); - } - } - - /** - * In Elasticsearch response, there is field changed between each query, such as "took". - * We have to replace those variants with fake constant to avoid re-generate documents. - * The order of fields in JSON is a little different from original because of internal - * key set in org.json. - */ - private String replaceChangingFields(String response) { - try { - JSONObject root = new JSONObject(response); - if (root.has("took")) { - root.put("took", 100); - } else { - return response; // return original response to minimize impact - } - - if (root.has("_shards")) { - JSONObject shards = root.getJSONObject("_shards"); - shards.put("total", 5); - shards.put("successful", 5); - } - return root.toString(); - } catch (JSONException e) { - // Response is not a valid JSON which is not our interest. - return response; - } - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/response/SqlResponseFormat.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/response/SqlResponseFormat.java deleted file mode 100644 index b38f413a48..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/response/SqlResponseFormat.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.doctest.core.response; - -import com.amazon.opendistroforelasticsearch.sql.utils.JsonPrettyFormatter; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; -import org.json.JSONArray; -import org.json.JSONObject; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -/** - * Different SQL response formats - */ -public enum SqlResponseFormat { - - IGNORE_RESPONSE { - @Override - public String format(SqlResponse sqlResponse) { - return ""; - } - }, - ORIGINAL_RESPONSE { - @Override - public String format(SqlResponse sqlResponse) { - return sqlResponse.body(); - } - }, - PRETTY_JSON_RESPONSE { - @Override - public String format(SqlResponse sqlResponse) { - String body = sqlResponse.body(); - try { - return JsonPrettyFormatter.format(body); - } catch (IOException e) { - throw new IllegalStateException( - StringUtils.format("Failed to pretty format response: %s", body), e); - } - } - }, - TABLE_RESPONSE { - @Override - public String format(SqlResponse sqlResponse) { - return format(sqlResponse, true); - } - }, - TABLE_UNSORTED_RESPONSE { - @Override - public String format(SqlResponse sqlResponse) { - return format(sqlResponse, false); - } - }; - - /** - * Format SQL response to specific format for documentation - * @param sqlResponse sql response - * @return string in specific format - */ - public abstract String format(SqlResponse sqlResponse); - - /** - * Note that we put this format() here because it's shared by two format enums. - * @param sqlResponse original response from plugin - * @param isSorted true to sort the result or just leave it as is - */ - protected String format(SqlResponse sqlResponse, boolean isSorted) { - JSONObject body = new JSONObject(sqlResponse.body()); - if (body.isNull("schema")) { - throw new IllegalStateException( - "Only JDBC response can be formatted to table: " + sqlResponse.body()); - } - - Object[] header = parseHeader(body.getJSONArray("schema")); - List rows = parseDataRows(body.getJSONArray("datarows"), isSorted); - - DataTable table = new DataTable(header); - for (Object[] row : rows) { - table.addRow(row); - } - return table.toString(); - } - - private Object[] parseHeader(JSONArray schema) { - Object[] header = new Object[schema.length()]; - for (int i = 0; i < header.length; i++) { - JSONObject nameType = schema.getJSONObject(i); - header[i] = nameType.optString("alias", nameType.getString("name")); - } - return header; - } - - private List parseDataRows(JSONArray rows, boolean isSorted) { - List rowsToSort = new ArrayList<>(); - for (Object row : rows) { - rowsToSort.add(((JSONArray) row).toList().toArray()); - } - - if (isSorted) { - sort(rowsToSort); - } - return rowsToSort; - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static void sort(List lists) { - lists.sort((list1, list2) -> { - if (list1 == null || list2 == null) { - return compareNullable(list1, list2); - } - - // Assume 2 lists are of same length and all elements are comparable - for (int i = 0; i < list1.length; i++) { - Comparable obj1 = (Comparable) list1[i]; - Comparable obj2 = (Comparable) list2[i]; - - if (obj1 == null || obj2 == null) { - return compareNullable(obj1, obj2); - } - - int result = obj1.compareTo(obj2); - if (result != 0) { - return result; - } - } - return 0; - }); - } - - /** Put NULL first (as smaller element) */ - private static int compareNullable(Object obj1, Object obj2) { - if (obj1 == null && obj2 == null) { - return 0; - } else if (obj1 == null) { - return -1; - } else { // obj2 == null - return 1; - } - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/DocBuilderTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/DocBuilderTest.java deleted file mode 100644 index 42b5466854..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/test/DocBuilderTest.java +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.doctest.core.test; - -import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.DocBuilder; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.markup.Document; -import org.apache.http.HttpEntity; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.RestClient; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.InOrder; -import org.mockito.Mock; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.runners.MockitoJUnitRunner; -import org.mockito.stubbing.Answer; - -import java.io.ByteArrayInputStream; -import java.io.IOException; - -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.inOrder; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * Test cases for {@link DocBuilder} - */ -@RunWith(MockitoJUnitRunner.class) -public class DocBuilderTest implements DocBuilder { - - private final String queryResponse = - "{\"schema\":[{\"name\":\"firstname\",\"type\":\"text\"}]," + - "\"datarows\":[[\"John\"]],\"total\":10,\"size\":1,\"status\":200}"; - - private final String explainResponse = - "{\"from\":0,\"size\":1,\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]}}"; - - @Mock - private Document document; - - private Verifier verifier; - - @Mock - private RestClient client; - - @Before - public void setUp() throws IOException { - when(document.section(any())).thenReturn(document); - when(document.subSection(any())).thenReturn(document); - when(document.paragraph(any())).thenReturn(document); - when(document.codeBlock(any(), any())).thenReturn(document); - when(document.table(any(), any())).thenReturn(document); - when(document.image(any(), any())).thenReturn(document); - verifier = new Verifier(document); - - when(client.performRequest(any())).then(new Answer() { - private int callCount = 0; - - @Override - public Response answer(InvocationOnMock invocationOnMock) throws IOException { - Response response = mock(Response.class); - HttpEntity entity = mock(HttpEntity.class); - when(response.getEntity()).thenReturn(entity); - - String body = (callCount++ == 0) ? queryResponse : explainResponse; - when(entity.getContent()).thenReturn(new ByteArrayInputStream(body.getBytes())); - return response; - } - }); - } - - @Test - public void sectionShouldIncludeTitleAndDescription() { - section( - title("Test"), - description("This is a test") - ); - - verifier.section("Test"). - subSection("Description"). - paragraph("This is a test"); - } - - @Test - public void sectionShouldIncludeMultiLineSql() { - section( - title("Test"), - description("This is a test"), - example( - description("This is an example for the test"), - post(multiLine( - "SELECT firstname", - "FROM accounts", - "WHERE age > 30") - ) - ) - ); - - verifier.section("Test"). - subSection("Description"). - paragraph("This is a test"). - subSection("Example"). - paragraph("This is an example for the test"). - codeBlock( - "SQL query", - "POST /_opendistro/_sql\n" + - "{\n" + - " \"query\" : \"\"\"\n" + - "\tSELECT firstname\n" + - "\tFROM accounts\n" + - "\tWHERE age > 30\n" + - "\t\"\"\"\n" + - "}" - ); - } - - @Test - public void sectionShouldIncludeExample() { - section( - title("Test"), - description("This is a test"), - images("rdd/querySyntax.png"), - example( - description("This is an example for the test"), - post("SELECT firstname FROM accounts") - ) - ); - - verifier.section("Test"). - subSection("Description"). - paragraph("This is a test"). - image("Rule ``querySyntax``", "/docs/user/img/rdd/querySyntax.png"). - subSection("Example"). - paragraph("This is an example for the test"). - codeBlock( - "SQL query", - "POST /_opendistro/_sql\n" + - "{\n" + - " \"query\" : \"SELECT firstname FROM accounts\"\n" + - "}" - ). - codeBlock( - "Explain", - "{\n" + - " \"from\" : 0,\n" + - " \"size\" : 1,\n" + - " \"_source\" : {\n" + - " \"includes\" : [\n" + - " \"firstname\"\n" + - " ],\n" + - " \"excludes\" : [ ]\n" + - " }\n" + - "}" - ).table( - "Result set", - "+---------+\n" + - "|firstname|\n" + - "+=========+\n" + - "| John|\n" + - "+---------+\n" - ); - } - - @Override - public RestClient restClient() { - return client; - } - - @Override - public Document openDocument() { - return document; - } - - private static class Verifier implements Document { - private final Document mock; - private final InOrder verifier; - - Verifier(Document mock) { - this.mock = mock; - this.verifier = inOrder(mock); - } - - @Override - public void close() { - verifier.verify(mock).close(); - } - - @Override - public Document section(String title) { - verifier.verify(mock).section(title); - return this; - } - - @Override - public Document subSection(String title) { - verifier.verify(mock).subSection(title); - return this; - } - - @Override - public Document paragraph(String text) { - verifier.verify(mock).paragraph(text); - return this; - } - - @Override - public Document codeBlock(String description, String code) { - verifier.verify(mock).codeBlock(description, code); - return this; - } - - @Override - public Document table(String description, String table) { - verifier.verify(mock).table(description, table); - return this; - } - - @Override - public Document image(String description, String filePath) { - verifier.verify(mock).image(description, filePath); - return this; - } - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/BasicQueryIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/BasicQueryIT.java deleted file mode 100644 index 94cd41918e..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/BasicQueryIT.java +++ /dev/null @@ -1,313 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.doctest.dql; - -import com.amazon.opendistroforelasticsearch.sql.doctest.core.DocTest; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.DocTestConfig; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.Example; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.Requests; - -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.KIBANA_REQUEST; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.TABLE_UNSORTED_RESPONSE; - -/** - * Doc test for basic SELECT query. - */ -@DocTestConfig(template = "dql/basics.rst", testData = {"accounts.json"}) -public class BasicQueryIT extends DocTest { - - @Section(1) - public void select() { - section( - title("SELECT"), - description("``SELECT`` clause specifies which fields in Elasticsearch index should be retrieved."), - images("rdd/selectElements.png", "rdd/selectElement.png"), - example( - title("Selecting All Fields"), - description( - "You can use ``*`` to fetch all fields in the index which is very convenient when you", - "just want to have a quick look at your data." - ), - post("SELECT * FROM accounts") - ), - example( - title("Selecting Specific Fields"), - description( - "More often you would give specific field name(s) in ``SELECT`` clause to", - "avoid large and unnecessary data retrieved." - ), - post("SELECT firstname, lastname FROM accounts") - ), - example( - title("Using Field Alias"), - description( - "Alias is often used to make your query more readable by giving your field a shorter name." - ), - post("SELECT account_number AS num FROM accounts") - ), - example( - title("Selecting Distinct Fields"), - description( - "``DISTINCT`` is useful when you want to de-duplicate and get unique field value.", - "You can provide one or more field names." - ), - post("SELECT DISTINCT age FROM accounts") - ) - ); - } - - @Section(2) - public void from() { - section( - title("FROM"), - description( - "``FROM`` clause specifies Elasticsearch index where the data should be retrieved from.", - "You've seen how to specify a single index in FROM clause in last section. Here we", - "provide examples for more use cases.\n\n" + - "Subquery in ``FROM`` clause is also supported. Please check out the documentation for more details." - ), - images("rdd/tableName.png"), - kibanaExample( - title("Using Index Alias"), - description( - "Similarly you can give index in ``FROM`` clause an alias and use it across clauses in query." - ), - post("SELECT acc.account_number FROM accounts acc") - ), - kibanaExample( - title("Selecting From Multiple Indices by Index Pattern"), - description( - "Alternatively you can query from multiple indices of similar names by index pattern.", - "This is very convenient for indices created by Logstash index template with date as suffix." - ), - post("SELECT account_number FROM account*") - ), - kibanaExample( - title("[Deprecating] Selecting From Specific Index Type"), - description( - "You can also specify type name explicitly though this has been deprecated in", - "later Elasticsearch version." - ), - post("SELECT account_number FROM accounts/account") - ) - ); - } - - @Section(3) - public void where() { - section( - title("WHERE"), - description( - "``WHERE`` clause specifies only Elasticsearch documents that meet the criteria should be affected.", - "It consists of predicates that uses ``=``, ``<>``, ``>``, ``>=``, ``<``, ``<=``, ``IN``,", - "``BETWEEN``, ``LIKE``, ``IS NULL`` or ``IS NOT NULL``. These predicates can be combined by", - "logical operator ``NOT``, ``AND`` or ``OR`` to build more complex expression.\n\n" + - "For ``LIKE`` and other full text search topics, please refer to Full Text Search documentation.\n\n" + - "Besides SQL query, WHERE clause can also be used in SQL statement such as ``DELETE``. Please refer to", - "Data Manipulation Language documentation for details." - ), - example( - title("Comparison Operators"), - description( - "Basic comparison operators, such as ``=``, ``<>``, ``>``, ``>=``, ``<``, ``<=``, can work for", - "number, string or date.", - "``IN`` and ``BETWEEN`` is convenient for comparison with multiple values or a range." - ), - post(multiLine( - "SELECT account_number", - "FROM accounts", - "WHERE account_number = 1" - )) - ), - example( - title("Missing Fields"), - description( - "As NoSQL database, Elasticsearch allows for flexible schema that documents in an index may have", - "different fields. In this case, you can use ``IS NULL`` or ``IS NOT NULL`` to retrieve missing", - "fields or existing fields only.\n\n" + - "Note that for now we don't differentiate missing field and field set to ``NULL`` explicitly." - ), - post(multiLine( - "SELECT account_number, employer", - "FROM accounts", - "WHERE employer IS NULL" - )) - ) - ); - } - - @Section(4) - public void groupBy() { - section( - title("GROUP BY"), - description( - "``GROUP BY`` groups documents with same field value into buckets. It is often used along with", - "aggregation functions to aggregate inside each bucket. Please refer to SQL Functions documentation", - "for more details.\n\n" + - "Note that ``WHERE`` clause is applied before ``GROUP BY`` clause." - ), - example( - title("Grouping by Fields"), - description(), - post(multiLine( - "SELECT age", - "FROM accounts", - "GROUP BY age" - )) - ), - example( - title("Grouping by Field Alias"), - description("Field alias is accessible in ``GROUP BY`` clause."), - post(multiLine( - "SELECT account_number AS num", - "FROM accounts", - "GROUP BY num" - )) - ), - example( - title("Grouping by Ordinal"), - description( - "Alternatively field ordinal in ``SELECT`` clause can be used too. However this is not", - "recommended because your ``GROUP BY`` clause depends on fields in ``SELECT`` clause", - "and require to change accordingly." - ), - post(multiLine( - "SELECT age", - "FROM accounts", - "GROUP BY 1" - )) - ), - example( - title("Grouping by Scalar Function"), - description( - "Scalar function can be used in ``GROUP BY`` clause and it's required to be present in", - "``SELECT`` clause too." - ), - post(multiLine( - "SELECT ABS(age) AS a", - "FROM accounts", - "GROUP BY ABS(age)" - )) - ) - ); - } - - @Section(5) - public void having() { - section( - title("HAVING"), - description( - "``HAVING`` clause filters result from ``GROUP BY`` clause by predicate(s). Because of this,", - "aggregation function, even different from those on ``SELECT`` clause, can be used in predicate." - ), - example( - description(), - post(multiLine( - "SELECT age, MAX(balance)", - "FROM accounts", - "GROUP BY age", - "HAVING MIN(balance) > 10000" - )) - ) - ); - } - - @Section(6) - public void orderBy() { - section( - title("ORDER BY"), - description("``ORDER BY`` clause specifies which fields used to sort the result and in which direction."), - orderByExample( - title("Ordering by Fields"), - description( - "Besides regular field names, ordinal, alias or scalar function can also be used similarly", - "as in ``GROUP BY``. ``ASC`` (by default) or ``DESC`` can be appended to indicate sorting in", - "ascending or descending order." - ), - post("SELECT account_number FROM accounts ORDER BY account_number DESC") - ), - orderByExample( - title("Specifying Order for Null"), - description( - "Additionally you can specify if documents with missing field be put first or last.", - "The default behavior of Elasticsearch is to return nulls or missing last.", - "You can make them present before non-nulls by using ``IS NOT NULL``." - ), - post(multiLine( - "SELECT employer", - "FROM accounts", - "ORDER BY employer IS NOT NULL" - )) - ) - ); - } - - @Section(7) - public void limit() { - section( - title("LIMIT"), - description( - "Mostly specifying maximum number of documents returned is necessary to prevent fetching", - "large amount of data into memory. `LIMIT` clause is helpful in this case." - ), - example( - title("Limiting Result Size"), - description( - "Given a positive number, ``LIMIT`` uses it as page size to fetch result of that size at most." - ), - post(multiLine( - "SELECT account_number", - "FROM accounts", - "ORDER BY account_number LIMIT 1" - )) - ), - example( - title("Fetching at Offset"), - description( - "Offset position can be given as first argument to indicate where to start fetching.", - "This can be used as simple pagination solution though it's inefficient on large index.", - "Generally ``ORDER BY`` is required in this case to ensure the same order between pages." - ), - post(multiLine( - "SELECT account_number", - "FROM accounts", - "ORDER BY account_number LIMIT 1, 1" - )) - ) - ); - } - - /** Document only Kibana request for example and ignore response as well as explain */ - private Example kibanaExample(String title, String description, Requests requests) { - return example(title, description, requests, - queryFormat(KIBANA_REQUEST, IGNORE_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ); - } - - /** Example for ORDER BY needs to maintain the order in original result */ - private Example orderByExample(String title, String description, Requests requests) { - return example(title, description, requests, - queryFormat(KIBANA_REQUEST, TABLE_UNSORTED_RESPONSE), - explainFormat(IGNORE_REQUEST, PRETTY_JSON_RESPONSE) - ); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/ComplexQueryIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/ComplexQueryIT.java deleted file mode 100644 index aa28b2ca21..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/ComplexQueryIT.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.doctest.dql; - -import com.amazon.opendistroforelasticsearch.sql.doctest.core.DocTest; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.DocTestConfig; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.Example; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.Requests; -import org.junit.Ignore; - -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.KIBANA_REQUEST; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.TABLE_RESPONSE; - -@DocTestConfig(template = "dql/complex.rst", testData = {"accounts.json", "employees_nested.json"}) -public class ComplexQueryIT extends DocTest { - - @Section(1) - public void subquery() { - section( - title("Subquery"), - description( - "A subquery is a complete ``SELECT`` statement which is used within another statement", - "and enclosed in parenthesis. From the explain output, you can notice that some subquery", - "are actually transformed to an equivalent join query to execute." - ), - /* - example( - title("Scalar Value Subquery"), - description( - "" - ), - post( - "SELECT firstname, lastname, balance " + - "FROM accounts " + - "WHERE balance >= ( " + - " SELECT AVG(balance) FROM accounts " + - ") " - ) - ),*/ - example( - title("Table Subquery"), - description(""), - post(multiLine( - "SELECT a1.firstname, a1.lastname, a1.balance", - "FROM accounts a1", - "WHERE a1.account_number IN (", - " SELECT a2.account_number", - " FROM accounts a2", - " WHERE a2.balance > 10000", - ")" - )) - ), - example( - title("Subquery in FROM Clause"), - description(""), - post(multiLine( - "SELECT a.f, a.l, a.a", - "FROM (", - " SELECT firstname AS f, lastname AS l, age AS a", - " FROM accounts", - " WHERE age > 30", - ") AS a" - )) - ) - ); - } - - @Section(2) - public void joins() { - section( - title("JOINs"), - description( - "A ``JOIN`` clause combines columns from one or more indices by using values common to each." - ), - images("rdd/tableSource.png", "rdd/joinPart.png"), - example( - title("Inner Join"), - description( - "Inner join is very commonly used that creates a new result set by combining columns", - "of two indices based on the join predicates specified. It iterates both indices and", - "compare each document to find all that satisfy the join predicates. Keyword ``JOIN``", - "is used and preceded by ``INNER`` keyword optionally. The join predicate(s) is specified", - "by ``ON`` clause.\n\n", - "Remark that the explain API output for join queries looks complicated. This is because", - "a join query is associated with two Elasticsearch DSL queries underlying and execute in", - "the separate query planner framework. You can interpret it by looking into the logical", - "plan and physical plan." - ), - post(multiLine( - "SELECT", - " a.account_number, a.firstname, a.lastname,", - " e.id, e.name", - "FROM accounts a", - "JOIN employees_nested e", - " ON a.account_number = e.id" - )) - ), - joinExampleWithoutExplain( - title("Cross Join"), - description( - "Cross join or Cartesian join combines each document from the first index with each from", - "the second. The result set is the Cartesian Product of documents from both indices.", - "It appears to be similar to inner join without ``ON`` clause to specify join condition.\n\n", - "Caveat: It is risky to do cross join even on two indices of medium size. This may trigger", - "our circuit breaker to terminate the query to avoid out of memory issue." - ), - post(multiLine( - "SELECT", - " a.account_number, a.firstname, a.lastname,", - " e.id, e.name", - "FROM accounts a", - "JOIN employees_nested e" - )) - ), - joinExampleWithoutExplain( - title("Outer Join"), - description( - "Outer join is used to retain documents from one or both indices although it does not satisfy", - "join predicate. For now, only ``LEFT OUTER JOIN`` is supported to retain rows from first index.", - "Note that keyword ``OUTER`` is optional." - ), - post(multiLine( - "SELECT", - " a.account_number, a.firstname, a.lastname,", - " e.id, e.name", - "FROM accounts a", - "LEFT JOIN employees_nested e", - " ON a.account_number = e.id" - )) - ) - ); - } - - @Ignore("Multi-query doesn't work for default format: https://github.com/opendistro-for-elasticsearch/sql/issues/388") - @Section(3) - public void setOperations() { - section( - title("Set Operations"), - description( - "Set operations allow results of multiple queries to be combined into a single result set.", - "The results to be combined are required to be of same type. In other word, they require to", - "have same column. Otherwise, a semantic analysis exception is raised." - ), - example( - title("UNION Operator"), - description( - "A ``UNION`` clause combines the results of two queries into a single result set. Duplicate rows", - "are removed unless ``UNION ALL`` clause is being used. A common use case of ``UNION`` is to combine", - "result set from data partitioned in indices daily or monthly." - ), - post(multiLine( - "SELECT balance, firstname, lastname", - "FROM accounts WHERE balance < 10000", - "UNION", - "SELECT balance, firstname, lastname", - "FROM accounts WHERE balance > 30000" - )) - ), - example( - title("MINUS Operator"), - description( - "A ``MINUS`` clause takes two queries too but returns resulting rows of first query that", - "do not appear in the other query. Duplicate rows are removed automatically as well." - ), - post(multiLine( - "SELECT balance, age", - "FROM accounts", - "WHERE balance < 10000", - "MINUS", - "SELECT balance, age", - "FROM accounts", - "WHERE age < 35" - )) - ) - ); - } - - private Example joinExampleWithoutExplain(String title, String description, Requests requests) { - return example(title, description, requests, - queryFormat(KIBANA_REQUEST, TABLE_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/MetaDataQueryIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/MetaDataQueryIT.java deleted file mode 100644 index 9850e2397f..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/MetaDataQueryIT.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.doctest.dql; - -import com.amazon.opendistroforelasticsearch.sql.doctest.core.DocTest; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.DocTestConfig; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.Example; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.Requests; - -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.KIBANA_REQUEST; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.TABLE_RESPONSE; - -@DocTestConfig(template = "dql/metadata.rst", testData = {"accounts.json", "employees_nested.json"}) -public class MetaDataQueryIT extends DocTest { - - @Section(1) - public void queryMetaData() { - section( - title("Querying Metadata"), - description( - "You can query your indices metadata by ``SHOW`` and ``DESCRIBE`` statement. These commands are", - "very useful for database management tool to enumerate all existing indices and get basic information", - "from the cluster." - ), - images("rdd/showStatement.png", "rdd/showFilter.png"), - metadataQueryExample( - title("Show All Indices Information"), - description( - "``SHOW`` statement lists all indices that match the search pattern. By using wildcard '%',", - "information for all indices in the cluster is returned." - ), - post("SHOW TABLES LIKE %") - ), - metadataQueryExample( - title("Show Specific Index Information"), - description("Here is an example that searches metadata for index name prefixed by 'acc'"), - post("SHOW TABLES LIKE acc%") - ), - metadataQueryExample( - title("Describe Index Fields Information"), - description("``DESCRIBE`` statement lists all fields for indices that can match the search pattern."), - post("DESCRIBE TABLES LIKE accounts") - ) - ); - } - - /** Explain doesn't work for SHOW/DESCRIBE so skip it */ - private Example metadataQueryExample(String title, String description, Requests requests) { - return example(title, description, requests, - queryFormat(KIBANA_REQUEST, TABLE_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/SQLFunctionsIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/SQLFunctionsIT.java deleted file mode 100644 index 29a1d6e7de..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/dql/SQLFunctionsIT.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.doctest.dql; - -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.function.ScalarFunction; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.DocTest; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.DocTestConfig; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; - -import static com.amazon.opendistroforelasticsearch.sql.antlr.semantic.types.TypeExpression.TypeExpressionSpec; - -@DocTestConfig(template = "dql/functions.rst") -public class SQLFunctionsIT extends DocTest { - - /** List only specifications of all SQL functions supported for now */ - @Section - public void listFunctions() { - for (ScalarFunction func : ScalarFunction.values()) { // Java Enum.values() return enums in order they are defined - section( - title(func.getName()), - description(listFunctionSpecs(func)) - ); - } - } - - private String listFunctionSpecs(ScalarFunction func) { - TypeExpressionSpec[] specs = func.specifications(); - if (specs.length == 0) { - return "Specification is undefined and type check is skipped for now"; - } - - StringBuilder specStr = new StringBuilder("Specifications: \n\n"); - for (int i = 0; i < specs.length; i++) { - specStr.append( - StringUtils.format("%d. %s%s\n", (i + 1), func.getName(), specs[i]) - ); - } - return specStr.toString(); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/interfaces/EndpointIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/interfaces/EndpointIT.java deleted file mode 100644 index c86963eb78..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/interfaces/EndpointIT.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.doctest.interfaces; - -import com.amazon.opendistroforelasticsearch.sql.doctest.core.DocTest; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.DocTestConfig; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; - -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.CURL_REQUEST; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; - -/** - * Doc test for endpoints to access the plugin. - */ -@DocTestConfig(template = "interfaces/endpoint.rst", testData = {"accounts.json"}) -public class EndpointIT extends DocTest { - - @Section(1) - public void queryByPost() { - section( - title("POST"), - description("You can also send HTTP POST request with your query in request body."), - example( - description(), - post("SELECT * FROM accounts"), - queryFormat(CURL_REQUEST, IGNORE_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ) - ); - } - - @Section(2) - public void explainQuery() { - section( - title("Explain"), - description( - "To translate your query, send it to explain endpoint. The explain output is Elasticsearch", - "domain specific language (DSL) in JSON format. You can just copy and paste it to your", - "console to run it against Elasticsearch directly." - ), - example( - description(), - post("SELECT firstname, lastname FROM accounts WHERE age > 20"), - queryFormat(IGNORE_REQUEST, IGNORE_RESPONSE), - explainFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE) - ) - ); - } - - @Section(3) - public void cursorQuery() { - section( - title("Cursor"), - description( - "To get paginated response for a query, user needs to provide `fetch_size` parameter as part of normal query.", - "The value of `fetch_size` should be greater than `0`. In absence of `fetch_size`, default value of 1000 is used.", - "A value of `0` will fallback to non-paginated response.", - "This feature is only available over `jdbc` format for now." - ), - example( - description(), - post("SELECT firstname, lastname FROM accounts WHERE age > 20 ORDER BY state ASC"), - queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ) - ); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/interfaces/ProtocolIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/interfaces/ProtocolIT.java deleted file mode 100644 index 30f3b82b5a..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/interfaces/ProtocolIT.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.doctest.interfaces; - -import com.amazon.opendistroforelasticsearch.sql.doctest.core.DocTest; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.DocTestConfig; -import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; - -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.CURL_REQUEST; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.IGNORE_RESPONSE; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.ORIGINAL_RESPONSE; -import static com.amazon.opendistroforelasticsearch.sql.doctest.core.response.SqlResponseFormat.PRETTY_JSON_RESPONSE; - -/** - * Doc test for plugin supported protocols. - */ -@DocTestConfig(template = "interfaces/protocol.rst", testData = {"accounts.json"}) -public class ProtocolIT extends DocTest { - - @Section(1) - public void requestFormat() { - section( - title("Request Format"), - description("The body of HTTP POST request can take a few more other fields with SQL query."), - example( - description( - "Use `filter` to add more conditions to Elasticsearch DSL directly." - ), - post( - body( - "\"query\": \"SELECT firstname, lastname, balance FROM accounts\"", - "\"filter\":{\"range\":{\"balance\":{\"lt\":10000}}}" - ) - ), - queryFormat(CURL_REQUEST, IGNORE_RESPONSE), - explainFormat(IGNORE_REQUEST, PRETTY_JSON_RESPONSE) - ), - example( - description("Use `parameters` for actual parameter value in prepared SQL query."), - post( - body( - "\"query\": \"SELECT * FROM accounts WHERE age = ?\"", - "\"parameters\": [{\"type\": \"integer\", \"value\": 30}]" - ) - ), - queryFormat(CURL_REQUEST, IGNORE_RESPONSE), - explainFormat(IGNORE_REQUEST, PRETTY_JSON_RESPONSE) - ) - ); - } - - @Section(2) - public void responseInJDBCFormat() { - section( - title("JDBC Format"), - description( - "By default the plugin return JDBC format. JDBC format is provided for JDBC driver and client side that needs both schema and", - "result set well formatted." - ), - example( - description( - "Here is an example for normal response. The `schema` includes field name and its type", - "and `datarows` includes the result set." - ), - post("SELECT firstname, lastname, age FROM accounts ORDER BY age LIMIT 2"), - queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ), - example( - description("If any error occurred, error message and the cause will be returned instead."), - post("SELECT unknown FROM accounts", params("format=jdbc")), - queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ) - ); - } - - @Section(3) - public void originalDSLResponse() { - section( - title("Elasticsearch DSL"), - description( - "The plugin returns original response from Elasticsearch in JSON. Because this is", - "the native response from Elasticsearch, extra efforts are needed to parse and interpret it." - ), - example( - description(), - post("SELECT firstname, lastname, age FROM accounts ORDER BY age LIMIT 2", params("format=json")), - queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ) - ); - } - - @Section(4) - public void responseInCSVFormat() { - section( - title("CSV Format"), - description("You can also use CSV format to download result set as CSV."), - example( - description(), - post("SELECT firstname, lastname, age FROM accounts ORDER BY age", params("format=csv")), - queryFormat(CURL_REQUEST, ORIGINAL_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ) - ); - } - - @Section(5) - public void responseInRawFormat() { - section( - title("Raw Format"), - description( - "Additionally raw format can be used to pipe the result to other command line tool for post processing." - ), - example( - description(), - post("SELECT firstname, lastname, age FROM accounts ORDER BY age", params("format=raw")), - queryFormat(CURL_REQUEST, ORIGINAL_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ) - ); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/AggregationExpressionIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/AggregationExpressionIT.java deleted file mode 100644 index 7d345cbf56..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/AggregationExpressionIT.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import org.json.JSONObject; -import org.junit.Ignore; -import org.junit.Test; - -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyDataRows; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifySchema; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.schema; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.rows; - -public class AggregationExpressionIT extends SQLIntegTestCase { - - @Override - protected void init() throws Exception { - loadIndex(Index.ACCOUNT); - loadIndex(Index.BANK); - } - - @Test - public void noGroupKeySingleFuncOverAggWithoutAliasShouldPass() { - JSONObject response = executeJdbcRequest(String.format( - "SELECT abs(MAX(age)) " + - "FROM %s", - Index.ACCOUNT.getName())); - - verifySchema(response, schema("abs(MAX(age))", null, "long")); - verifyDataRows(response, rows(40)); - } - - @Test - public void noGroupKeyMaxAddMinShouldPass() { - JSONObject response = executeJdbcRequest(String.format( - "SELECT MAX(age) + MIN(age) as add " + - "FROM %s", - Index.ACCOUNT.getName())); - - verifySchema(response, schema("add", "add", "long")); - verifyDataRows(response, rows(60)); - } - - // todo age field should has long type instead of integer type. - @Ignore - @Test - public void noGroupKeyMaxAddLiteralShouldPass() { - JSONObject response = executeJdbcRequest(String.format( - "SELECT MAX(age) + 1 as add " + - "FROM %s", - Index.ACCOUNT.getName())); - - verifySchema(response, schema("add", "add", "long")); - verifyDataRows(response, rows(41)); - } - - @Test - public void noGroupKeyAvgOnIntegerShouldPass() { - JSONObject response = executeJdbcRequest(String.format( - "SELECT AVG(age) as avg " + - "FROM %s", - Index.BANK.getName())); - - verifySchema(response, schema("avg", "avg", "double")); - verifyDataRows(response, rows(34)); - } - - @Test - public void hasGroupKeyAvgOnIntegerShouldPass() { - JSONObject response = executeJdbcRequest(String.format( - "SELECT gender, AVG(age) as avg " + - "FROM %s " + - "GROUP BY gender", - Index.BANK.getName())); - - verifySchema(response, - schema("gender", null, "text"), - schema("avg", "avg", "double")); - verifyDataRows(response, - rows("m", 34.25), - rows("f", 33.666666666666664d)); - } - - @Test - public void hasGroupKeyMaxAddMinShouldPass() { - JSONObject response = executeJdbcRequest(String.format( - "SELECT gender, MAX(age) + MIN(age) as add " + - "FROM %s " + - "GROUP BY gender", - Index.ACCOUNT.getName())); - - verifySchema(response, - schema("gender", null, "text"), - schema("add", "add", "long")); - verifyDataRows(response, - rows("m", 60), - rows("f", 60)); - } - - // todo age field should has long type instead of integer type. - @Ignore - @Test - public void hasGroupKeyMaxAddLiteralShouldPass() { - JSONObject response = executeJdbcRequest(String.format( - "SELECT gender, MAX(age) + 1 as add " + - "FROM %s " + - "GROUP BY gender", - Index.ACCOUNT.getName())); - - verifySchema(response, - schema("gender", null, "text"), - schema("add", "add", "long")); - verifyDataRows(response, - rows("m", 1), - rows("f", 1)); - } - - @Test - public void noGroupKeyLogMaxAddMinShouldPass() { - JSONObject response = executeJdbcRequest(String.format( - "SELECT Log(MAX(age) + MIN(age)) as log " + - "FROM %s", - Index.ACCOUNT.getName())); - - verifySchema(response, schema("log", "log", "double")); - verifyDataRows(response, rows(4.0943445622221d)); - } - - @Test - public void hasGroupKeyLogMaxAddMinShouldPass() { - JSONObject response = executeJdbcRequest(String.format( - "SELECT gender, Log(MAX(age) + MIN(age)) as log " + - "FROM %s " + - "GROUP BY gender", - Index.ACCOUNT.getName())); - - verifySchema(response, - schema("gender", null, "text"), - schema("log", "log", "double")); - verifyDataRows(response, - rows("m", 4.0943445622221d), - rows("f", 4.0943445622221d)); - } - - // todo age field should has long type instead of integer type. - @Ignore - @Test - public void AddLiteralOnGroupKeyShouldPass() { - JSONObject response = executeJdbcRequest(String.format( - "SELECT gender, age+10, max(balance) as max " + - "FROM %s " + - "WHERE gender = 'm' and age < 22 " + - "GROUP BY gender, age " + - "ORDER BY age", - Index.ACCOUNT.getName())); - - verifySchema(response, - schema("gender", null, "text"), - schema("age", "age", "long"), - schema("max", "max", "long")); - verifyDataRows(response, - rows("m", 30, 49568), - rows("m", 31, 49433)); - } - - @Test - public void logWithAddLiteralOnGroupKeyShouldPass() { - JSONObject response = executeJdbcRequest(String.format( - "SELECT gender, Log(age+10) as logAge, max(balance) as max " + - "FROM %s " + - "WHERE gender = 'm' and age < 22 " + - "GROUP BY gender, age " + - "ORDER BY age", - Index.ACCOUNT.getName())); - - verifySchema(response, - schema("gender", null, "text"), - schema("logAge", "logAge", "double"), - schema("max", "max", "long")); - verifyDataRows(response, - rows("m", 3.4011973816621555d, 49568), - rows("m", 3.4339872044851463d, 49433)); - } - - // todo max field should has long as type instead of integer type. - @Ignore - @Test - public void logWithAddLiteralOnGroupKeyAndMaxSubtractLiteralShouldPass() { - JSONObject response = executeJdbcRequest(String.format( - "SELECT gender, Log(age+10) as logAge, max(balance) - 100 as max " + - "FROM %s " + - "WHERE gender = 'm' and age < 22 " + - "GROUP BY gender, age " + - "ORDER BY age", - Index.ACCOUNT.getName())); - - verifySchema(response, - schema("gender", null, "text"), - schema("logAge", "logAge", "double"), - schema("max", "max", "long")); - verifyDataRows(response, - rows("m", 3.4011973816621555d, 49468), - rows("m", 3.4339872044851463d, 49333)); - } - - private JSONObject executeJdbcRequest(String query) { - return new JSONObject(executeQuery(query, "jdbc")); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/AggregationIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/AggregationIT.java deleted file mode 100644 index 23480051c5..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/AggregationIT.java +++ /dev/null @@ -1,1258 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import org.json.JSONArray; -import org.json.JSONObject; -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Locale; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_ACCOUNT; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_DOG; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_GAME_OF_THRONES; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_NESTED_TYPE; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_ONLINE; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.rows; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.schema; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyDataRows; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyDataRowsInOrder; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifySchema; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.isEmptyString; -import static org.hamcrest.Matchers.not; - -public class AggregationIT extends SQLIntegTestCase { - - @Override - protected void init() throws Exception { - loadIndex(Index.ACCOUNT); - loadIndex(Index.GAME_OF_THRONES); - loadIndex(Index.DOG); - loadIndex(Index.ONLINE); - loadIndex(Index.NESTED); - } - - @Test - public void countTest() throws IOException { - - JSONObject result = executeQuery(String.format("SELECT COUNT(*) FROM %s", TEST_INDEX_ACCOUNT)); - Assert.assertThat(getTotalHits(result), equalTo(1000)); - Assert.assertThat(getIntAggregationValue(result, "COUNT(*)", "value"), equalTo(1000)); - } - - @Test - public void countDistinctTest() { - JSONObject response = executeJdbcRequest(String.format("SELECT COUNT(distinct gender) FROM %s", TEST_INDEX_ACCOUNT)); - - verifySchema(response, schema("COUNT(DISTINCT gender)", null, "integer")); - verifyDataRows(response, rows(2)); - } - - @Test - public void countWithDocsHintTest() throws Exception { - - JSONObject result = executeQuery(String.format("SELECT /*! DOCS_WITH_AGGREGATION(10) */ count(*) from %s", - TEST_INDEX_ACCOUNT)); - JSONArray hits = (JSONArray)result.query("/hits/hits"); - Assert.assertThat(hits.length(), equalTo(10)); - } - - @Test - public void sumTest() throws IOException { - - JSONObject result = executeQuery(String.format("SELECT SUM(balance) FROM %s", TEST_INDEX_ACCOUNT)); - Assert.assertThat(getTotalHits(result), equalTo(1000)); - Assert.assertThat(getDoubleAggregationValue(result, "SUM(balance)", "value"), equalTo(25714837.0)); - } - - @Test - public void minTest() throws IOException { - - JSONObject result = executeQuery(String.format("SELECT MIN(age) FROM %s", TEST_INDEX_ACCOUNT)); - Assert.assertThat(getTotalHits(result), equalTo(1000)); - Assert.assertThat(getDoubleAggregationValue(result, "MIN(age)", "value"), equalTo(20.0)); - } - - @Test - public void maxTest() throws IOException { - - JSONObject result = executeQuery(String.format("SELECT MAX(age) FROM %s", TEST_INDEX_ACCOUNT)); - Assert.assertThat(getTotalHits(result), equalTo(1000)); - Assert.assertThat(getDoubleAggregationValue(result, "MAX(age)", "value"), equalTo(40.0)); - } - - @Test - public void avgTest() throws IOException { - - JSONObject result = executeQuery(String.format("SELECT AVG(age) FROM %s", TEST_INDEX_ACCOUNT)); - Assert.assertThat(getTotalHits(result), equalTo(1000)); - Assert.assertThat(getDoubleAggregationValue(result, "AVG(age)", "value"), equalTo(30.171)); - } - - @Test - public void statsTest() throws IOException { - - JSONObject result = executeQuery(String.format("SELECT STATS(age) FROM %s", TEST_INDEX_ACCOUNT)); - Assert.assertThat(getTotalHits(result), equalTo(1000)); - Assert.assertThat(getIntAggregationValue(result, "STATS(age)", "count"), equalTo(1000)); - Assert.assertThat(getDoubleAggregationValue(result, "STATS(age)", "min"), equalTo(20.0)); - Assert.assertThat(getDoubleAggregationValue(result, "STATS(age)", "max"), equalTo(40.0)); - Assert.assertThat(getDoubleAggregationValue(result, "STATS(age)", "avg"), equalTo(30.171)); - Assert.assertThat(getDoubleAggregationValue(result, "STATS(age)", "sum"), equalTo(30171.0)); - } - - @Test - public void extendedStatsTest() throws IOException { - - JSONObject result = executeQuery(String.format("SELECT EXTENDED_STATS(age) FROM %s", - TEST_INDEX_ACCOUNT)); - Assert.assertThat(getTotalHits(result), equalTo(1000)); - Assert.assertThat(getDoubleAggregationValue(result, "EXTENDED_STATS(age)", "min"), equalTo(20.0)); - Assert.assertThat(getDoubleAggregationValue(result, "EXTENDED_STATS(age)", "max"), equalTo(40.0)); - Assert.assertThat(getDoubleAggregationValue(result, "EXTENDED_STATS(age)", "avg"), equalTo(30.171)); - Assert.assertThat(getDoubleAggregationValue(result, "EXTENDED_STATS(age)", "sum"), equalTo(30171.0)); - Assert.assertThat(getDoubleAggregationValue(result, "EXTENDED_STATS(age)", "sum_of_squares"), equalTo(946393.0)); - Assert.assertEquals(6.008640362012022, getDoubleAggregationValue(result, "EXTENDED_STATS(age)", "std_deviation"), 0.0001); - Assert.assertEquals(36.10375899999996, getDoubleAggregationValue(result, "EXTENDED_STATS(age)", "variance"), 0.0001); - } - - @Test - public void percentileTest() throws IOException { - - JSONObject result = executeQuery(String.format("SELECT PERCENTILES(age) FROM %s", TEST_INDEX_ACCOUNT)); - Assert.assertThat(getTotalHits(result), equalTo(1000)); - Assert.assertEquals(20.0, getDoubleAggregationValue(result, "PERCENTILES(age)", "values", "1.0"), 0.001); - Assert.assertEquals(21.0, getDoubleAggregationValue(result, "PERCENTILES(age)", "values", "5.0"), 0.001); - Assert.assertEquals(25.0, getDoubleAggregationValue(result, "PERCENTILES(age)", "values", "25.0"), 0.001); - // All percentiles are approximations calculated by t-digest, however, P50 has the widest distribution (not sure why) - Assert.assertEquals(30.5, getDoubleAggregationValue(result, "PERCENTILES(age)", "values", "50.0"), 0.6); - Assert.assertEquals(35.0, getDoubleAggregationValue(result, "PERCENTILES(age)", "values", "75.0"), 0.001); - Assert.assertEquals(39.0, getDoubleAggregationValue(result, "PERCENTILES(age)", "values", "95.0"), 0.001); - Assert.assertEquals(40.0, getDoubleAggregationValue(result, "PERCENTILES(age)", "values", "99.0"), 0.001); - } - - @Test - public void percentileTestSpecific() throws IOException { - - JSONObject result = executeQuery(String.format("SELECT PERCENTILES(age,25.0,75.0) FROM %s", - TEST_INDEX_ACCOUNT)); - - Assert.assertThat(getTotalHits(result), equalTo(1000)); - Assert.assertEquals(25.0, getDoubleAggregationValue(result, "PERCENTILES(age,25.0,75.0)", "values", "25.0"), 0.001); - Assert.assertEquals(35.0, getDoubleAggregationValue(result, "PERCENTILES(age,25.0,75.0)", "values", "75.0"), 0.001); - } - - @Test - public void aliasTest() throws IOException { - - JSONObject result = executeQuery(String.format("SELECT COUNT(*) AS mycount FROM %s", - TEST_INDEX_ACCOUNT)); - Assert.assertThat(getTotalHits(result), equalTo(1000)); - Assert.assertThat(getIntAggregationValue(result, "mycount", "value"), equalTo(1000)); - } - - @Test - public void groupByTest() throws Exception { - JSONObject result = executeQuery(String.format("SELECT COUNT(*) FROM %s GROUP BY gender", - TEST_INDEX_ACCOUNT)); - assertResultForGroupByTest(result); - } - - @Test - public void groupByUsingTableAliasTest() throws Exception { - JSONObject result = executeQuery(String.format("SELECT COUNT(*) FROM %s a GROUP BY a.gender", - TEST_INDEX_ACCOUNT)); - assertResultForGroupByTest(result); - } - - @Test - public void groupByUsingTableNamePrefixTest() throws Exception { - JSONObject result = executeQuery(String.format( - "SELECT COUNT(*) FROM %s GROUP BY elasticsearch-sql_test_index_account.gender", - TEST_INDEX_ACCOUNT - )); - assertResultForGroupByTest(result); - } - - private void assertResultForGroupByTest(JSONObject result) { - Assert.assertThat(getTotalHits(result), equalTo(1000)); - JSONObject gender = getAggregation(result, "gender"); - Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); - - final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); - final int maleBucketId = isMaleFirst ? 0 : 1; - final int femaleBucketId = isMaleFirst ? 1 : 0; - - final String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); - final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); - - Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); - Assert.assertThat(gender.query(maleBucketPrefix + "/COUNT(*)/value"), equalTo(507)); - Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); - Assert.assertThat(gender.query(femaleBucketPrefix + "/COUNT(*)/value"), equalTo(493)); - } - - @Test - public void groupByHavingTest() throws Exception { - JSONObject result = executeQuery(String.format( - "SELECT gender " + - "FROM %s " + - "GROUP BY gender " + - "HAVING COUNT(*) > 0", TEST_INDEX_ACCOUNT)); - assertResultForGroupByHavingTest(result); - } - - @Test - public void groupByHavingUsingTableAliasTest() throws Exception { - JSONObject result = executeQuery(String.format( - "SELECT a.gender " + - "FROM %s a " + - "GROUP BY a.gender " + - "HAVING COUNT(*) > 0", TEST_INDEX_ACCOUNT)); - assertResultForGroupByHavingTest(result); - } - - @Test - public void groupByHavingUsingTableNamePrefixTest() throws Exception { - JSONObject result = executeQuery(String.format( - "SELECT elasticsearch-sql_test_index_account.gender " + - "FROM %s " + - "GROUP BY elasticsearch-sql_test_index_account.gender " + - "HAVING COUNT(*) > 0", TEST_INDEX_ACCOUNT)); - assertResultForGroupByHavingTest(result); - } - - private void assertResultForGroupByHavingTest(JSONObject result) { - Assert.assertThat(getTotalHits(result), equalTo(1000)); - JSONObject gender = getAggregation(result, "gender"); - Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); - - final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); - final int maleBucketId = isMaleFirst ? 0 : 1; - final int femaleBucketId = isMaleFirst ? 1 : 0; - - final String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); - final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); - - Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); - Assert.assertThat(gender.query(maleBucketPrefix + "/count_0/value"), equalTo(507)); - Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); - Assert.assertThat(gender.query(femaleBucketPrefix + "/count_0/value"), equalTo(493)); - } - - @Ignore //todo VerificationException: table alias or field name missing - @Test - public void groupBySubqueryTest() throws Exception { - - JSONObject result = executeQuery(String.format( - "SELECT COUNT(*) FROM %s " + - "WHERE firstname IN (SELECT firstname FROM %s) " + - "GROUP BY gender", - TEST_INDEX_ACCOUNT, TEST_INDEX_ACCOUNT)); - Assert.assertThat(getTotalHits(result), equalTo(1000)); - JSONObject gender = getAggregation(result, "gender"); - Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); - - final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); - final int maleBucketId = isMaleFirst ? 0 : 1; - final int femaleBucketId = isMaleFirst ? 1 : 0; - - final String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); - final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); - - Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); - Assert.assertThat(gender.query(maleBucketPrefix + "/COUNT(*)/value"), equalTo(507)); - Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); - Assert.assertThat(gender.query(femaleBucketPrefix + "/COUNT(*)/value"), equalTo(493)); - } - - @Test - public void postFilterTest() throws Exception { - - JSONObject result = executeQuery(String.format("SELECT /*! POST_FILTER({\\\"term\\\":" + - "{\\\"gender\\\":\\\"m\\\"}}) */ COUNT(*) FROM %s GROUP BY gender", - TEST_INDEX_ACCOUNT)); - Assert.assertThat(getTotalHits(result), equalTo(507)); - JSONObject gender = getAggregation(result, "gender"); - Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); - - final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); - final int maleBucketId = isMaleFirst ? 0 : 1; - final int femaleBucketId = isMaleFirst ? 1 : 0; - - final String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); - final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); - - Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); - Assert.assertThat(gender.query(maleBucketPrefix + "/COUNT(*)/value"), equalTo(507)); - Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); - Assert.assertThat(gender.query(femaleBucketPrefix + "/COUNT(*)/value"), equalTo(493)); - } - - @Test - public void multipleGroupByTest() throws Exception { - - JSONObject result = executeQuery(String.format("SELECT COUNT(*) FROM %s GROUP BY gender," + - " terms('field'='age','size'=200,'alias'='age')", - TEST_INDEX_ACCOUNT)); - Assert.assertThat(getTotalHits(result), equalTo(1000)); - JSONObject gender = getAggregation(result, "gender"); - Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); - - final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); - final int maleBucketId = isMaleFirst ? 0 : 1; - final int femaleBucketId = isMaleFirst ? 1 : 0; - - final String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); - final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); - - final JSONArray mAgeBuckets = (JSONArray)(gender.optQuery(maleBucketPrefix + "/age/buckets")); - final JSONArray fAgeBuckets = (JSONArray)(gender.optQuery(femaleBucketPrefix + "/age/buckets")); - - final Set expectedAges = IntStream.range(20, 41).boxed().collect(Collectors.toCollection(HashSet::new)); - Assert.assertThat(mAgeBuckets.length(), equalTo(expectedAges.size())); - Assert.assertThat(fAgeBuckets.length(), equalTo(expectedAges.size())); - - final Set actualAgesM = new HashSet<>(expectedAges.size()); - final Set actualAgesF = new HashSet<>(expectedAges.size()); - mAgeBuckets.iterator().forEachRemaining(json -> actualAgesM.add(((JSONObject)json).getInt("key"))); - fAgeBuckets.iterator().forEachRemaining(json -> actualAgesF.add(((JSONObject)json).getInt("key"))); - - Assert.assertThat(actualAgesM, equalTo(expectedAges)); - Assert.assertThat(actualAgesF, equalTo(expectedAges)); - } - - @Test - public void multipleGroupBysWithSize() throws Exception { - - JSONObject result = executeQuery(String.format("SELECT COUNT(*) FROM %s GROUP BY gender," + - " terms('alias'='ageAgg','field'='age','size'=3)", - TEST_INDEX_ACCOUNT)); - Assert.assertThat(getTotalHits(result), equalTo(1000)); - JSONObject gender = getAggregation(result, "gender"); - Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); - - final JSONArray mAgeBuckets = (JSONArray)(gender.optQuery("/buckets/0/ageAgg/buckets")); - final JSONArray fAgeBuckets = (JSONArray)(gender.optQuery("/buckets/0/ageAgg/buckets")); - - Assert.assertThat(mAgeBuckets.length(), equalTo(3)); - Assert.assertThat(fAgeBuckets.length(), equalTo(3)); - } - - @Test - public void termsWithSize() throws Exception { - - JSONObject result = executeQuery(String.format("SELECT COUNT(*) FROM %s GROUP BY terms" + - "('alias'='ageAgg','field'='age','size'=3)", - TEST_INDEX_ACCOUNT)); - Assert.assertThat(getTotalHits(result), equalTo(1000)); - JSONObject gender = getAggregation(result, "ageAgg"); - Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(3)); - } - - @Test - public void termsWithMissing() throws Exception { - - JSONObject result = executeQuery(String.format("SELECT count(*) FROM %s GROUP BY terms" + - "('alias'='nick','field'='nickname','missing'='no_nickname')", - TEST_INDEX_GAME_OF_THRONES)); - JSONObject nick = getAggregation(result, "nick"); - - Optional noNicknameBucket = Optional.empty(); - Iterator iter = nick.getJSONArray("buckets").iterator(); - while (iter.hasNext()) { - JSONObject bucket = (JSONObject)iter.next(); - if (bucket.getString("key").equals("no_nickname")) { - noNicknameBucket = Optional.of(bucket); - Assert.assertThat(bucket.getInt("doc_count"), equalTo(6)); - } - } - Assert.assertTrue(noNicknameBucket.isPresent()); - } - - @Test - public void termsWithOrder() throws Exception { - - final String dog1 = "snoopy"; - final String dog2 = "rex"; - - JSONObject result = executeQuery(String.format("SELECT count(*) FROM %s GROUP BY terms" + - "('field'='dog_name', 'alias'='dog_name', 'order'='desc')", - TEST_INDEX_DOG)); - JSONObject dogName = getAggregation(result, "dog_name"); - - String firstDog = (String)(dogName.optQuery("/buckets/0/key")); - String secondDog = (String)(dogName.optQuery("/buckets/1/key")); - Assert.assertThat(firstDog, equalTo(dog1)); - Assert.assertThat(secondDog, equalTo(dog2)); - - result = executeQuery(String.format("SELECT count(*) FROM %s GROUP BY terms" + - "('field'='dog_name', 'alias'='dog_name', 'order'='asc')", TEST_INDEX_DOG)); - - dogName = getAggregation(result, "dog_name"); - - firstDog = (String)(dogName.optQuery("/buckets/0/key")); - secondDog = (String)(dogName.optQuery("/buckets/1/key")); - Assert.assertThat(firstDog, equalTo(dog2)); - Assert.assertThat(secondDog, equalTo(dog1)); - } - - @Test - public void orderByAscTest() { - JSONObject response = executeJdbcRequest(String.format("SELECT COUNT(*) FROM %s " + - "GROUP BY gender ORDER BY COUNT(*)", TEST_INDEX_ACCOUNT)); - - verifySchema(response, schema("COUNT(*)", null, "integer")); - verifyDataRows(response, - rows(493), - rows(507)); - } - - @Test - public void orderByAliasAscTest() { - JSONObject response = executeJdbcRequest(String.format("SELECT COUNT(*) as count FROM %s " + - "GROUP BY gender ORDER BY count", TEST_INDEX_ACCOUNT)); - - verifySchema(response, schema("count", "count", "integer")); - verifyDataRowsInOrder(response, - rows(493), - rows(507)); - } - - @Test - public void orderByDescTest() throws IOException { - JSONObject response = executeJdbcRequest(String.format("SELECT COUNT(*) FROM %s " + - "GROUP BY gender ORDER BY COUNT(*) DESC", TEST_INDEX_ACCOUNT)); - - verifySchema(response, schema("COUNT(*)", null, "integer")); - verifyDataRowsInOrder(response, - rows(507), - rows(493)); - } - - @Test - public void orderByAliasDescTest() throws IOException { - JSONObject response = executeJdbcRequest(String.format("SELECT COUNT(*) as count FROM %s " + - "GROUP BY gender ORDER BY count DESC", TEST_INDEX_ACCOUNT)); - - verifySchema(response, schema("count", "count", "integer")); - verifyDataRowsInOrder(response, - rows(507), - rows(493)); - } - - @Test - public void limitTest() throws IOException { - JSONObject response = executeJdbcRequest(String.format("SELECT COUNT(*) FROM %s " + - "GROUP BY age ORDER BY COUNT(*) LIMIT 5", TEST_INDEX_ACCOUNT)); - - verifySchema(response, schema("COUNT(*)", null, "integer")); - verifyDataRowsInOrder(response, - rows(35), - rows(39), - rows(39), - rows(42), - rows(42)); - } - - @Test - public void countGroupByRange() throws IOException { - - JSONObject result = executeQuery(String.format("SELECT COUNT(age) FROM %s" + - " GROUP BY range(age, 20,25,30,35,40)", TEST_INDEX_ACCOUNT)); - JSONObject ageAgg = getAggregation(result, "range(age,20,25,30,35,40)"); - JSONArray buckets = ageAgg.getJSONArray("buckets"); - Assert.assertThat(buckets.length(), equalTo(4)); - - final int[] expectedResults = new int[] {225, 226, 259, 245}; - - for (int i = 0; i < expectedResults.length; ++i) { - - Assert.assertThat(buckets.query(String.format(Locale.ROOT, "/%d/COUNT(age)/value", i)), - equalTo(expectedResults[i])); - } - } - - /** - * http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-aggregations-bucket-datehistogram-aggregation.html - */ - @Test - public void countGroupByDateTest() throws IOException { - - String result = explainQuery(String.format("select insert_time from %s group by date_histogram" + - "('field'='insert_time','fixed_interval'='1h','format'='yyyy-MM','min_doc_count'=5) ", TEST_INDEX_ONLINE)); - Assert.assertThat(result.replaceAll("\\s+", ""), - containsString("{\"date_histogram\":{\"field\":\"insert_time\",\"format\":\"yyyy-MM\"," + - "\"fixed_interval\":\"1h\",\"offset\":0,\"order\":{\"_key\":\"asc\"},\"keyed\":false," + - "\"min_doc_count\":5}")); - } - - @Test - public void countGroupByDateTestWithAlias() throws IOException { - String result = explainQuery(String.format("select insert_time from %s group by date_histogram" + - "('field'='insert_time','fixed_interval'='1h','format'='yyyy-MM','alias'='myAlias')", TEST_INDEX_ONLINE)); - Assert.assertThat(result.replaceAll("\\s+",""), - containsString("myAlias\":{\"date_histogram\":{\"field\":\"insert_time\"," + - "\"format\":\"yyyy-MM\",\"fixed_interval\":\"1h\"")); - } - -// /** -// * http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-aggregations-bucket-daterange-aggregation.html -// */ -// @Test -// public void countDateRangeTest() throws IOException, SqlParseException, SQLFeatureNotSupportedException { -// String result = explainQuery(String.format("select online from %s group by date_range(field='insert_time'," + -// "'format'='yyyy-MM-dd' ,'2014-08-18','2014-08-17','now-8d','now-7d','now-6d','now')", -// TEST_INDEX_ONLINE)); -// // TODO: fix the query or fix the code for the query to work -// } - - @Test - public void topHitTest() throws IOException { - - String query = String.format("select topHits('size'=3,age='desc') from %s group by gender", TEST_INDEX_ACCOUNT); - JSONObject result = executeQuery(query); - JSONObject gender = getAggregation(result, "gender"); - Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); - - final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); - final int maleBucketId = isMaleFirst ? 0 : 1; - final int femaleBucketId = isMaleFirst ? 1 : 0; - - final String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); - final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); - - Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); - Assert.assertThat(gender.query(maleBucketPrefix + "/topHits(size=3,age=desc)/hits/total/value"),equalTo(507)); - Assert.assertThat(gender.query(maleBucketPrefix + "/topHits(size=3,age=desc)/hits/total/relation"),equalTo("eq")); - Assert.assertThat(((JSONArray)gender.query(maleBucketPrefix + "/topHits(size=3,age=desc)/hits/hits")).length(), - equalTo(3)); - Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); - Assert.assertThat(gender.query(femaleBucketPrefix + "/topHits(size=3,age=desc)/hits/total/value"), equalTo(493)); - Assert.assertThat(gender.query(femaleBucketPrefix + "/topHits(size=3,age=desc)/hits/total/relation"), equalTo("eq")); - Assert.assertThat(((JSONArray)gender.query(femaleBucketPrefix + "/topHits(size=3,age=desc)/hits/hits")).length(), - equalTo(3)); - } - - @Test - public void topHitTest_WithInclude() throws IOException { - - String query = String.format("select topHits('size'=3,age='desc','include'=age) from %s group by gender", - TEST_INDEX_ACCOUNT); - JSONObject result = executeQuery(query); - JSONObject gender = getAggregation(result, "gender"); - Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); - - final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); - final int maleBucketId = isMaleFirst ? 0 : 1; - final int femaleBucketId = isMaleFirst ? 1 : 0; - - final String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); - final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); - - Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); - Assert.assertThat(gender.query(maleBucketPrefix + "/topHits(size=3,age=desc,include=age)/hits/total/value"), - equalTo(507)); - Assert.assertThat(gender.query(maleBucketPrefix + "/topHits(size=3,age=desc,include=age)/hits/total/relation"), - equalTo("eq")); - Assert.assertThat(((JSONArray)gender.query( - maleBucketPrefix + "/topHits(size=3,age=desc,include=age)/hits/hits")).length(), - equalTo(3)); - - Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); - Assert.assertThat(gender.query(femaleBucketPrefix + "/topHits(size=3,age=desc,include=age)/hits/total/value"), - equalTo(493)); - Assert.assertThat(gender.query(femaleBucketPrefix + "/topHits(size=3,age=desc,include=age)/hits/total/relation"), - equalTo("eq")); - Assert.assertThat(((JSONArray)gender.query( - femaleBucketPrefix + "/topHits(size=3,age=desc,include=age)/hits/hits")).length(), - equalTo(3)); - - for (int i = 0; i < 2; ++i) { - for (int j = 0; j < 3; ++j) { - JSONObject source = (JSONObject)gender.query(String.format(Locale.ROOT, - "/buckets/%d/topHits(size=3,age=desc,include=age)/hits/hits/%d/_source", i, j)); - Assert.assertThat(source.length(), equalTo(1)); - Assert.assertTrue(source.has("age")); - Assert.assertThat(source.getInt("age"), equalTo(40)); - } - } - } - - @Test - public void topHitTest_WithIncludeTwoFields() throws IOException { - - String query = String.format("select topHits('size'=3,'include'='age,firstname',age='desc') from %s " + - "group by gender", TEST_INDEX_ACCOUNT); - JSONObject result = executeQuery(query); - JSONObject gender = getAggregation(result, "gender"); - Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); - - for (int i = 0; i < 2; ++i) { - for (int j = 0; j < 3; ++j) { - JSONObject source = (JSONObject)gender.query(String.format(Locale.ROOT, - "/buckets/%d/topHits(size=3,include=age,firstname,age=desc)/hits/hits/%d/_source", i, j)); - Assert.assertThat(source.length(), equalTo(2)); - Assert.assertTrue(source.has("age")); - Assert.assertThat(source.getInt("age"), equalTo(40)); - Assert.assertTrue(source.has("firstname")); - final String name = source.getString("firstname"); - Assert.assertThat(name, not(isEmptyString())); - } - } - } - - @Test - public void topHitTest_WithExclude() throws IOException { - - String query = String.format("select topHits('size'=3,'exclude'='lastname',age='desc') from " + - "%s group by gender", TEST_INDEX_ACCOUNT); - JSONObject result = executeQuery(query); - JSONObject gender = getAggregation(result, "gender"); - Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); - - final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); - final int maleBucketId = isMaleFirst ? 0 : 1; - final int femaleBucketId = isMaleFirst ? 1 : 0; - - final String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); - final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); - - Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); - Assert.assertThat(gender.query(maleBucketPrefix + "/topHits(size=3,exclude=lastname,age=desc)/hits/total/value"), - equalTo(507)); - Assert.assertThat(gender.query(maleBucketPrefix + "/topHits(size=3,exclude=lastname,age=desc)/hits/total/relation"), - equalTo("eq")); - Assert.assertThat(((JSONArray)gender.query( - maleBucketPrefix + "/topHits(size=3,exclude=lastname,age=desc)/hits/hits")).length(), - equalTo(3)); - - Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); - Assert.assertThat(gender.query(femaleBucketPrefix + "/topHits(size=3,exclude=lastname,age=desc)/hits/total/value"), - equalTo(493)); - Assert.assertThat(gender.query(femaleBucketPrefix + "/topHits(size=3,exclude=lastname,age=desc)/hits/total/relation"), - equalTo("eq")); - Assert.assertThat(((JSONArray)gender.query( - femaleBucketPrefix + "/topHits(size=3,exclude=lastname,age=desc)/hits/hits")).length(), - equalTo(3)); - - final Set expectedFields = new HashSet<>(Arrays.asList( - "account_number", - "firstname", - "address", - "balance", - "gender", - "city", - "employer", - "state", - "age", - "email" - )); - - for (int i = 0; i < 2; ++i) { - for (int j = 0; j < 3; ++j) { - JSONObject source = (JSONObject)gender.query(String.format(Locale.ROOT, - "/buckets/%d/topHits(size=3,exclude=lastname,age=desc)/hits/hits/%d/_source", i, j)); - Assert.assertThat(source.length(), equalTo(expectedFields.size())); - Assert.assertFalse(source.has("lastname")); - Assert.assertThat(source.keySet().containsAll(expectedFields), equalTo(true)); - } - } - } - - //region not migrated - - // script on metric aggregation tests. uncomment if your elastic has scripts enable (disabled by default) -// @Test -// public void sumWithScriptTest() throws IOException, SqlParseException, SQLFeatureNotSupportedException { -// Aggregations result = query(String.format("SELECT SUM(script('','doc[\\'balance\\'].value + doc[\\'balance\\'].value')) as doubleSum FROM %s", TEST_INDEX)); -// Sum sum = result.get("doubleSum"); -// assertThat(sum.getValue(), equalTo(25714837.0*2)); -// } -// -// @Test -// public void sumWithImplicitScriptTest() throws IOException, SqlParseException, SQLFeatureNotSupportedException { -// Aggregations result = query(String.format("SELECT SUM(balance + balance) as doubleSum FROM %s", TEST_INDEX)); -// Sum sum = result.get("doubleSum"); -// assertThat(sum.getValue(), equalTo(25714837.0*2)); -// } -// -// @Test -// public void sumWithScriptTestNoAlias() throws IOException, SqlParseException, SQLFeatureNotSupportedException { -// Aggregations result = query(String.format("SELECT SUM(balance + balance) FROM %s", TEST_INDEX)); -// Sum sum = result.get("SUM(script=script(balance + balance,doc('balance').value + doc('balance').value))"); -// assertThat(sum.getValue(), equalTo(25714837.0*2)); -// } -// -// @Test -// public void scriptedMetricAggregation() throws SQLFeatureNotSupportedException, SqlParseException { -// Aggregations result = query ("select scripted_metric('map_script'='if(doc[\\'balance\\'].value > 49670){ if(!_agg.containsKey(\\'ages\\')) { _agg.put(\\'ages\\',doc[\\'age\\'].value); } " + -// "else { _agg.put(\\'ages\\',_agg.get(\\'ages\\')+doc[\\'age\\'].value); }}'," + -// "'reduce_script'='sumThem = 0; for (a in _aggs) { if(a.containsKey(\\'ages\\')){ sumThem += a.get(\\'ages\\');} }; return sumThem;') as wierdSum from " + TEST_INDEX + ""); -// ScriptedMetric metric = result.get("wierdSum"); -// Assert.assertEquals(136L,metric.aggregation()); -// } -// -// @Test -// public void scriptedMetricConcatWithStringParamAndReduceParamAggregation() throws SQLFeatureNotSupportedException, SqlParseException { -// String query = "select scripted_metric(\n" + -// " 'init_script' = '_agg[\"concat\"]=[] ',\n" + -// " 'map_script'='_agg.concat.add(doc[field].value)' ,\n" + -// " 'combine_script'='return _agg.concat.join(delim);',\t\t\t\t\n" + -// " 'reduce_script'='_aggs.removeAll(\"\"); return _aggs.join(delim)'," + -// "'@field' = 'name.firstname' , '@delim'=';',@reduce_delim =';' ) as all_characters \n" + -// "from "+TEST_INDEX+""; -// Aggregations result = query (query); -// ScriptedMetric metric = result.get("all_characters"); -// List names = Arrays.asList(metric.aggregation().toString().split(";")); -// -// -// Assert.assertEquals(4,names.size()); -// String[] expectedNames = new String[]{"brandon","daenerys","eddard","jaime"}; -// for(String name : expectedNames){ -// Assert.assertTrue("not contains:" + name,names.contains(name)); -// } -// } -// -// @Test -// public void scriptedMetricAggregationWithNumberParams() throws SQLFeatureNotSupportedException, SqlParseException { -// Aggregations result = query ("select scripted_metric('map_script'='if(doc[\\'balance\\'].value > 49670){ if(!_agg.containsKey(\\'ages\\')) { _agg.put(\\'ages\\',doc[\\'age\\'].value+x); } " + -// "else { _agg.put(\\'ages\\',_agg.get(\\'ages\\')+doc[\\'age\\'].value+x); }}'," + -// "'reduce_script'='sumThem = 0; for (a in _aggs) { if(a.containsKey(\\'ages\\')){ sumThem += a.get(\\'ages\\');} }; return sumThem;'" + -// ",'@x'=3) as wierdSum from " + TEST_INDEX + ""); -// ScriptedMetric metric = result.get("wierdSum"); -// Assert.assertEquals(148L,metric.aggregation()); -// } -// - -// @Test -// public void topHitTest_WithIncludeAndExclude() throws IOException, SqlParseException, SQLFeatureNotSupportedException { -// Aggregations result = query(String.format("select topHits('size'=3,'exclude'='lastname','include'='firstname,lastname',age='desc') from %s group by gender ", TEST_INDEX_ACCOUNT)); -// List buckets = ((Terms) (result.asList().get(0))).getBuckets(); -// for (Terms.Bucket bucket : buckets) { -// SearchHits hits = ((InternalTopHits) bucket.getAggregations().asList().get(0)).getHits(); -// for (SearchHit hit : hits) { -// Set fields = hit.getSourceAsMap().keySet(); -// Assert.assertEquals(1, fields.size()); -// Assert.assertTrue(fields.contains("firstname")); -// } -// } -// } -// -// private Aggregations query(String query) throws SqlParseException, SQLFeatureNotSupportedException { -// SqlElasticSearchRequestBuilder select = getSearchRequestBuilder(query); -// return ((SearchResponse)select.get()).getAggregations(); -// } -// -// private SqlElasticSearchRequestBuilder getSearchRequestBuilder(String query) throws SqlParseException, SQLFeatureNotSupportedException { -// SearchDao searchDao = MainTestSuite.getSearchDao(); -// return (SqlElasticSearchRequestBuilder) searchDao.explain(query).explain(); -// } -// -// @Test -// public void testFromSizeWithAggregations() throws Exception { -// final String query1 = String.format("SELECT /*! DOCS_WITH_AGGREGATION(0,1) */" + -// " account_number FROM %s GROUP BY gender", TEST_INDEX_ACCOUNT); -// SearchResponse response1 = (SearchResponse) getSearchRequestBuilder(query1).get(); -// -// Assert.assertEquals(1, response1.getHits().getHits().length); -// Terms gender1 = response1.getAggregations().get("gender"); -// Assert.assertEquals(2, gender1.getBuckets().size()); -// Object account1 = response1.getHits().getHits()[0].getSourceAsMap().get("account_number"); -// -// final String query2 = String.format("SELECT /*! DOCS_WITH_AGGREGATION(1,1) */" + -// " account_number FROM %s GROUP BY gender", TEST_INDEX_ACCOUNT); -// SearchResponse response2 = (SearchResponse) getSearchRequestBuilder(query2).get(); -// -// Assert.assertEquals(1, response2.getHits().getHits().length); -// Terms gender2 = response2.getAggregations().get("gender"); -// Assert.assertEquals(2, gender2.getBuckets().size()); -// Object account2 = response2.getHits().getHits()[0].getSourceAsMap().get("account_number"); -// -// Assert.assertEquals(response1.getHits().getTotalHits(), response2.getHits().getTotalHits()); -// Assert.assertNotEquals(account1, account2); -// } -// -// @Test -// public void testSubAggregations() throws Exception { -// Set expectedAges = new HashSet<>(ContiguousSet.create(Range.closed(20, 40), DiscreteDomain.integers())); -// final String query = String.format("SELECT /*! DOCS_WITH_AGGREGATION(10) */" + -// " * FROM %s GROUP BY (gender, terms('field'='age','size'=200,'alias'='age')), (state) LIMIT 200,200", TEST_INDEX_ACCOUNT); -// -// Map> buckets = new HashMap<>(); -// -// SqlElasticSearchRequestBuilder select = getSearchRequestBuilder(query); -// SearchResponse response = (SearchResponse) select.get(); -// Aggregations result = response.getAggregations(); -// -// Terms gender = result.get("gender"); -// for(Terms.Bucket genderBucket : gender.getBuckets()) { -// String genderKey = genderBucket.getKey().toString(); -// buckets.put(genderKey, new HashSet()); -// Terms ageBuckets = (Terms) genderBucket.getAggregations().get("age"); -// for(Terms.Bucket ageBucket : ageBuckets.getBuckets()) { -// buckets.get(genderKey).add(Integer.parseInt(ageBucket.getKey().toString())); -// } -// } -// -// Assert.assertEquals(2, buckets.keySet().size()); -// Assert.assertEquals(expectedAges, buckets.get("m")); -// Assert.assertEquals(expectedAges, buckets.get("f")); -// -// Terms state = result.get("state.keyword"); -// for(Terms.Bucket stateBucket : state.getBuckets()) { -// if(stateBucket.getKey().toString().equalsIgnoreCase("ak")) { -// Assert.assertTrue("There are 22 entries for state ak", stateBucket.getDocCount() == 22); -// } -// } -// -// Assert.assertEquals(response.getHits().getTotalHits(), 1000); -// Assert.assertEquals(response.getHits().getHits().length, 10); -// } -// -// @Test -// public void testSimpleSubAggregations() throws Exception { -// final String query = String.format("SELECT /*! DOCS_WITH_AGGREGATION(10) */ * FROM %s GROUP BY (gender), (state) ", TEST_INDEX_ACCOUNT); -// -// SqlElasticSearchRequestBuilder select = getSearchRequestBuilder(query); -// SearchResponse response = (SearchResponse) select.get(); -// Aggregations result = response.getAggregations(); -// -// Terms gender = result.get("gender"); -// for(Terms.Bucket genderBucket : gender.getBuckets()) { -// String genderKey = genderBucket.getKey().toString(); -// Assert.assertTrue("Gender should be m or f", genderKey.equals("m") || genderKey.equals("f")); -// } -// -// Assert.assertEquals(2, gender.getBuckets().size()); -// -// Terms state = result.get("state.keyword"); -// for(Terms.Bucket stateBucket : state.getBuckets()) { -// if(stateBucket.getKey().toString().equalsIgnoreCase("ak")) { -// Assert.assertTrue("There are 22 entries for state ak", stateBucket.getDocCount() == 22); -// } -// } -// -// Assert.assertEquals(response.getHits().getTotalHits(), 1000); -// Assert.assertEquals(response.getHits().getHits().length, 10); -// } -// -// @Test -// public void geoHashGrid() throws SQLFeatureNotSupportedException, SqlParseException { -// Aggregations result = query(String.format("SELECT COUNT(*) FROM %s/location GROUP BY geohash_grid(field='center',precision=5) ", TEST_INDEX_LOCATION)); -// InternalGeoHashGrid grid = result.get("geohash_grid(field=center,precision=5)"); -// Collection buckets = grid.getBuckets(); -// for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) { -// Assert.assertTrue(bucket.getKeyAsString().equals("w2fsm") || bucket.getKeyAsString().equals("w0p6y") ); -// Assert.assertEquals(1,bucket.getDocCount()); -// } -// } -// -// @Test -// public void geoBounds() throws SQLFeatureNotSupportedException, SqlParseException { -// Aggregations result = query(String.format("SELECT * FROM %s/location GROUP BY geo_bounds(field='center',alias='bounds') ", TEST_INDEX_LOCATION)); -// InternalGeoBounds bounds = result.get("bounds"); -// Assert.assertEquals(0.5,bounds.bottomRight().getLat(),0.001); -// Assert.assertEquals(105.0,bounds.bottomRight().getLon(),0.001); -// Assert.assertEquals(5.0,bounds.topLeft().getLat(),0.001); -// Assert.assertEquals(100.5,bounds.topLeft().getLon(),0.001); -// } -// -// @Test -// public void groupByOnNestedFieldTest() throws Exception { -// Aggregations result = query(String.format("SELECT COUNT(*) FROM %s GROUP BY nested(message.info)", TEST_INDEX_NESTED_TYPE)); -// InternalNested nested = result.get("message.info@NESTED"); -// Terms infos = nested.getAggregations().get("message.info"); -// Assert.assertEquals(3,infos.getBuckets().size()); -// for(Terms.Bucket bucket : infos.getBuckets()) { -// String key = bucket.getKey().toString(); -// long count = ((ValueCount) bucket.getAggregations().get("COUNT(*)")).getValue(); -// if(key.equalsIgnoreCase("a")) { -// Assert.assertEquals(2, count); -// } -// else if(key.equalsIgnoreCase("c")) { -// Assert.assertEquals(2, count); -// } -// else if(key.equalsIgnoreCase("b")) { -// Assert.assertEquals(1, count); -// } -// else { -// throw new Exception(String.format("Unexpected key. expected: a OR b OR c . found: %s", key)); -// } -// } -// } -// -// @Test -// public void groupByTestWithFilter() throws Exception { -// Aggregations result = query(String.format("SELECT COUNT(*) FROM %s GROUP BY filter(gender='m'),gender", TEST_INDEX_ACCOUNT)); -// InternalFilter filter = result.get("filter(gender = 'm')@FILTER"); -// Terms gender = filter.getAggregations().get("gender"); -// -// for(Terms.Bucket bucket : gender.getBuckets()) { -// String key = bucket.getKey().toString(); -// long count = ((ValueCount) bucket.getAggregations().get("COUNT(*)")).getValue(); -// if(key.equalsIgnoreCase("m")) { -// Assert.assertEquals(507, count); -// } -// else { -// throw new Exception(String.format("Unexpected key. expected: only m. found: %s", key)); -// } -// } -// } -// -// - //endregion not migrated - - @Test - public void groupByOnNestedFieldWithFilterTest() throws Exception { - - String query = String.format("SELECT COUNT(*) FROM %s GROUP BY nested(message.info)," + - "filter('myFilter',message.info = 'a')", TEST_INDEX_NESTED_TYPE); - JSONObject result = executeQuery(query); - - JSONObject aggregation = getAggregation(result, "message.info@NESTED"); - JSONArray buckets = (JSONArray)aggregation.optQuery("/myFilter@FILTER/message.info/buckets"); - Assert.assertNotNull(buckets); - Assert.assertThat(buckets.length(), equalTo(1)); - - JSONObject bucket = buckets.getJSONObject(0); - Assert.assertThat(bucket.getString("key"), equalTo("a")); - Assert.assertThat(bucket.query("/COUNT(*)/value"), equalTo(2)); - } - - @Test - public void minOnNestedField() throws Exception { - - String query = String.format("SELECT min(nested(message.dayOfWeek)) as minDays FROM %s", - TEST_INDEX_NESTED_TYPE); - JSONObject result = executeQuery(query); - JSONObject aggregation = getAggregation(result, "message.dayOfWeek@NESTED"); - Assert.assertEquals(1.0, (double)aggregation.query("/minDays/value"), 0.0001); - } - - @Test - public void sumOnNestedField() throws Exception { - - String query = String.format("SELECT sum(nested(message.dayOfWeek)) as sumDays FROM %s", - TEST_INDEX_NESTED_TYPE); - JSONObject result = executeQuery(query); - JSONObject aggregation = getAggregation(result, "message.dayOfWeek@NESTED"); - Assert.assertEquals(19.0, (double)aggregation.query("/sumDays/value"), 0.0001); - } - - @Test - public void histogramOnNestedField() throws Exception { - - String query = String.format("select count(*) from %s group by histogram" + - "('field'='message.dayOfWeek','nested'='message','interval'='2' , 'alias' = 'someAlias' )", - TEST_INDEX_NESTED_TYPE); - JSONObject result = executeQuery(query); - JSONObject aggregation = getAggregation(result, "message@NESTED"); - - final Map expectedCountsByKey = new HashMap<>(); - expectedCountsByKey.put(0.0, 2); - expectedCountsByKey.put(2.0, 1); - expectedCountsByKey.put(4.0, 2); - expectedCountsByKey.put(6.0, 1); - - JSONArray buckets = (JSONArray)aggregation.query("/someAlias/buckets"); - Assert.assertThat(buckets.length(), equalTo(4)); - - buckets.forEach(obj -> { - JSONObject bucket = (JSONObject)obj; - final double key = bucket.getDouble("key"); - Assert.assertTrue(expectedCountsByKey.containsKey(key)); - Assert.assertThat(bucket.getJSONObject("COUNT(*)").getInt("value"), - equalTo(expectedCountsByKey.get(key))); - }); - } - - @Test - public void reverseToRootGroupByOnNestedFieldWithFilterTestWithReverseNestedAndEmptyPath() throws Exception { - - String query = String.format("SELECT COUNT(*) FROM %s GROUP BY nested(message.info)," + - "filter('myFilter',message.info = 'a'),reverse_nested(someField,'')", TEST_INDEX_NESTED_TYPE); - JSONObject result = executeQuery(query); - JSONObject aggregation = getAggregation(result, "message.info@NESTED"); - - JSONArray msgInfoBuckets = (JSONArray)aggregation.optQuery("/myFilter@FILTER/message.info/buckets"); - Assert.assertNotNull(msgInfoBuckets); - Assert.assertThat(msgInfoBuckets.length(), equalTo(1)); - - JSONArray someFieldBuckets = (JSONArray)msgInfoBuckets.optQuery("/0/someField@NESTED/someField/buckets"); - Assert.assertNotNull(msgInfoBuckets); - Assert.assertThat(msgInfoBuckets.length(), equalTo(1)); - Assert.assertThat(someFieldBuckets.query("/0/key"), equalTo("b")); - Assert.assertThat(someFieldBuckets.query("/0/COUNT(*)/value"), equalTo(2)); - } - - @Test - public void reverseToRootGroupByOnNestedFieldWithFilterTestWithReverseNestedNoPath() throws Exception { - - String query = String.format("SELECT COUNT(*) FROM %s GROUP BY nested(message.info),filter" + - "('myFilter',message.info = 'a'),reverse_nested(someField)", TEST_INDEX_NESTED_TYPE); - JSONObject result = executeQuery(query); - JSONObject aggregation = getAggregation(result, "message.info@NESTED"); - - JSONArray msgInfoBuckets = (JSONArray)aggregation.optQuery("/myFilter@FILTER/message.info/buckets"); - Assert.assertNotNull(msgInfoBuckets); - Assert.assertThat(msgInfoBuckets.length(), equalTo(1)); - - JSONArray someFieldBuckets = (JSONArray)msgInfoBuckets.optQuery("/0/someField@NESTED/someField/buckets"); - Assert.assertNotNull(msgInfoBuckets); - Assert.assertThat(someFieldBuckets.length(), equalTo(1)); - Assert.assertThat(someFieldBuckets.query("/0/key"), equalTo("b")); - Assert.assertThat(someFieldBuckets.query("/0/COUNT(*)/value"), equalTo(2)); - } - - @Test - public void reverseToRootGroupByOnNestedFieldWithFilterTestWithReverseNestedOnHistogram() throws Exception { - - String query = String.format("SELECT COUNT(*) FROM %s GROUP BY nested(message.info)," + - "filter('myFilter',message.info = 'a'),histogram('field'='myNum','reverse_nested'='','interval'='2', " + - "'alias' = 'someAlias' )", TEST_INDEX_NESTED_TYPE); - JSONObject result = executeQuery(query); - JSONObject aggregation = getAggregation(result, "message.info@NESTED"); - - JSONArray msgInfoBuckets = (JSONArray)aggregation.optQuery("/myFilter@FILTER/message.info/buckets"); - Assert.assertNotNull(msgInfoBuckets); - Assert.assertThat(msgInfoBuckets.length(), equalTo(1)); - - JSONArray someAliasBuckets = (JSONArray)msgInfoBuckets.optQuery("/0/someAlias@NESTED/someAlias/buckets"); - Assert.assertNotNull(msgInfoBuckets); - Assert.assertThat(someAliasBuckets.length(), equalTo(3)); - - final Map expectedCountsByKey = new HashMap<>(); - expectedCountsByKey.put(0.0, 1); - expectedCountsByKey.put(2.0, 0); - expectedCountsByKey.put(4.0, 1); - - someAliasBuckets.forEach(obj -> { - JSONObject bucket = (JSONObject)obj; - final double key = bucket.getDouble("key"); - Assert.assertTrue(expectedCountsByKey.containsKey(key)); - Assert.assertThat(bucket.getJSONObject("COUNT(*)").getInt("value"), - equalTo(expectedCountsByKey.get(key))); - }); - } - - @Test - public void reverseToRootGroupByOnNestedFieldWithFilterAndSumOnReverseNestedField() throws Exception { - - String query = String.format("SELECT sum(reverse_nested(myNum)) bla FROM %s GROUP BY " + - "nested(message.info),filter('myFilter',message.info = 'a')", TEST_INDEX_NESTED_TYPE); - JSONObject result = executeQuery(query); - JSONObject aggregation = getAggregation(result, "message.info@NESTED"); - - JSONArray msgInfoBuckets = (JSONArray)aggregation.optQuery("/myFilter@FILTER/message.info/buckets"); - Assert.assertNotNull(msgInfoBuckets); - Assert.assertThat(msgInfoBuckets.length(), equalTo(1)); - - Assert.assertNotNull(msgInfoBuckets.optQuery("/0/myNum@NESTED/bla/value")); - JSONObject bla = (JSONObject)msgInfoBuckets.query("/0/myNum@NESTED/bla"); - Assert.assertEquals(5.0, bla.getDouble("value"), 0.000001); - } - - @Test - public void reverseAnotherNestedGroupByOnNestedFieldWithFilterTestWithReverseNestedNoPath() throws Exception { - - String query = String.format("SELECT COUNT(*) FROM %s GROUP BY nested(message.info)," + - "filter('myFilter',message.info = 'a'),reverse_nested(comment.data,'~comment')", - TEST_INDEX_NESTED_TYPE); - JSONObject result = executeQuery(query); - JSONObject aggregation = getAggregation(result, "message.info@NESTED"); - - JSONArray msgInfoBuckets = (JSONArray)aggregation.optQuery("/myFilter@FILTER/message.info/buckets"); - Assert.assertNotNull(msgInfoBuckets); - Assert.assertThat(msgInfoBuckets.length(), equalTo(1)); - - JSONArray commentDataBuckets = (JSONArray)msgInfoBuckets.optQuery("/0/comment.data@NESTED_REVERSED" + - "/comment.data@NESTED/comment.data/buckets"); - Assert.assertNotNull(commentDataBuckets); - Assert.assertThat(commentDataBuckets.length(), equalTo(1)); - Assert.assertThat(commentDataBuckets.query("/0/key"), equalTo("ab")); - Assert.assertThat(commentDataBuckets.query("/0/COUNT(*)/value"), equalTo(2)); - } - - @Test - public void reverseAnotherNestedGroupByOnNestedFieldWithFilterTestWithReverseNestedOnHistogram() throws Exception { - - String query = String.format("SELECT COUNT(*) FROM %s GROUP BY nested(message.info),filter" + - "('myFilter',message.info = 'a'),histogram('field'='comment.likes','reverse_nested'='~comment'," + - "'interval'='2' , 'alias' = 'someAlias' )", TEST_INDEX_NESTED_TYPE); - JSONObject result = executeQuery(query); - JSONObject aggregation = getAggregation(result, "message.info@NESTED"); - - JSONArray msgInfoBuckets = (JSONArray)aggregation.optQuery("/myFilter@FILTER/message.info/buckets"); - Assert.assertNotNull(msgInfoBuckets); - Assert.assertThat(msgInfoBuckets.length(), equalTo(1)); - - JSONArray someAliasBuckets = (JSONArray)msgInfoBuckets.optQuery( - "/0/~comment@NESTED_REVERSED/~comment@NESTED/someAlias/buckets"); - Assert.assertNotNull(msgInfoBuckets); - Assert.assertThat(someAliasBuckets.length(), equalTo(2)); - - final Map expectedCountsByKey = new HashMap<>(); - expectedCountsByKey.put(0.0, 1); - expectedCountsByKey.put(2.0, 1); - - someAliasBuckets.forEach(obj -> { - JSONObject bucket = (JSONObject)obj; - final double key = bucket.getDouble("key"); - Assert.assertTrue(expectedCountsByKey.containsKey(key)); - Assert.assertThat(bucket.getJSONObject("COUNT(*)").getInt("value"), - equalTo(expectedCountsByKey.get(key))); - }); - } - - @Test - public void reverseAnotherNestedGroupByOnNestedFieldWithFilterAndSumOnReverseNestedField() throws Exception { - - String query = String.format("SELECT sum(reverse_nested(comment.likes,'~comment')) bla FROM %s " + - "GROUP BY nested(message.info),filter('myFilter',message.info = 'a')", TEST_INDEX_NESTED_TYPE); - JSONObject result = executeQuery(query); - JSONObject aggregation = getAggregation(result, "message.info@NESTED"); - - JSONArray msgInfoBuckets = (JSONArray)aggregation.optQuery("/myFilter@FILTER/message.info/buckets"); - Assert.assertNotNull(msgInfoBuckets); - Assert.assertThat(msgInfoBuckets.length(), equalTo(1)); - - Assert.assertNotNull(msgInfoBuckets.optQuery( - "/0/comment.likes@NESTED_REVERSED/comment.likes@NESTED/bla/value")); - JSONObject bla = (JSONObject)msgInfoBuckets.query("/0/comment.likes@NESTED_REVERSED/comment.likes@NESTED/bla"); - Assert.assertEquals(4.0, bla.getDouble("value"), 0.000001); - } - - @Test - public void docsReturnedTestWithoutDocsHint() throws Exception { - String query = String.format("SELECT count(*) from %s", TEST_INDEX_ACCOUNT); - JSONObject result = executeQuery(query); - Assert.assertThat(getHits(result).length(), equalTo(0)); - } - - @Test - public void docsReturnedTestWithDocsHint() throws Exception { - String query = String.format("SELECT /*! DOCS_WITH_AGGREGATION(10) */ count(*) from %s", - TEST_INDEX_ACCOUNT); - JSONObject result = executeQuery(query); - Assert.assertThat(getHits(result).length(), equalTo(10)); - } - - @Ignore("There is not any text field in the index. Need fix later") - @Test - public void termsWithScript() throws Exception { - String query = String.format("select count(*), avg(all_client) from %s group by terms('alias'='asdf'," + - " substring(field, 0, 1)), date_histogram('alias'='time', 'field'='timestamp', " + - "'interval'='20d ', 'format'='yyyy-MM-dd') limit 1000", TEST_INDEX_ONLINE); - String result = explainQuery(query); - - Assert.assertThat(result, containsString("\"script\":{\"source\"")); - Assert.assertThat(result, containsString("substring(0, 1)")); - } - - @Test - public void groupByScriptedDateHistogram() throws Exception { - String query = String.format("select count(*), avg(all_client) from %s group by date_histogram('alias'='time'," + - " ceil(all_client), 'fixed_interval'='20d ', 'format'='yyyy-MM-dd') limit 1000" , TEST_INDEX_ONLINE); - String result = explainQuery(query); - - Assert.assertThat(result, containsString("Math.ceil(doc['all_client'].value);")); - Assert.assertThat(result, containsString("\"script\":{\"source\"")); - } - - @Test - public void groupByScriptedHistogram() throws Exception { - String query = String.format("select count(*) from %s group by histogram('alias'='all_field', pow(all_client,1))", - TEST_INDEX_ONLINE); - String result = explainQuery(query); - - Assert.assertThat(result, containsString("Math.pow(doc['all_client'].value, 1)")); - Assert.assertThat(result, containsString("\"script\":{\"source\"")); - } - - @Test - public void distinctWithOneField() { - Assert.assertEquals( - executeQuery("SELECT DISTINCT name.lastname FROM " + TEST_INDEX_GAME_OF_THRONES, "jdbc"), - executeQuery("SELECT name.lastname FROM " + TEST_INDEX_GAME_OF_THRONES - + " GROUP BY name.lastname", "jdbc") - ); - } - - @Test - public void distinctWithMultipleFields() { - Assert.assertEquals( - executeQuery("SELECT DISTINCT age, gender FROM " + TEST_INDEX_ACCOUNT, "jdbc"), - executeQuery("SELECT age, gender FROM " + TEST_INDEX_ACCOUNT - + " GROUP BY age, gender", "jdbc") - ); - } - - private JSONObject getAggregation(final JSONObject queryResult, final String aggregationName) - { - final String aggregationsObjName = "aggregations"; - Assert.assertTrue(queryResult.has(aggregationsObjName)); - - final JSONObject aggregations = queryResult.getJSONObject(aggregationsObjName); - Assert.assertTrue(aggregations.has(aggregationName)); - return aggregations.getJSONObject(aggregationName); - } - - private int getIntAggregationValue(final JSONObject queryResult, final String aggregationName, - final String fieldName) { - - final JSONObject targetAggregation = getAggregation(queryResult, aggregationName); - Assert.assertTrue(targetAggregation.has(fieldName)); - return targetAggregation.getInt(fieldName); - } - - private double getDoubleAggregationValue(final JSONObject queryResult, final String aggregationName, - final String fieldName) { - - final JSONObject targetAggregation = getAggregation(queryResult, aggregationName); - Assert.assertTrue(targetAggregation.has(fieldName)); - return targetAggregation.getDouble(fieldName); - } - - private double getDoubleAggregationValue(final JSONObject queryResult, final String aggregationName, - final String fieldName, final String subFieldName) { - - final JSONObject targetAggregation = getAggregation(queryResult, aggregationName); - Assert.assertTrue(targetAggregation.has(fieldName)); - final JSONObject targetField = targetAggregation.getJSONObject(fieldName); - Assert.assertTrue(targetField.has(subFieldName)); - - return targetField.getDouble(subFieldName); - } - - private JSONObject executeJdbcRequest(String query) { - return new JSONObject(executeQuery(query, "jdbc")); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/CsvFormatResponseIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/CsvFormatResponseIT.java deleted file mode 100644 index c7b06710ee..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/CsvFormatResponseIT.java +++ /dev/null @@ -1,778 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import com.amazon.opendistroforelasticsearch.sql.executor.csv.CSVResult; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.Response; -import org.hamcrest.Matcher; -import org.hamcrest.core.AnyOf; -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.LinkedList; -import java.util.List; -import java.util.Locale; -import java.util.stream.Collectors; - -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_ACCOUNT; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_DOG; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_GAME_OF_THRONES; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_NESTED_TYPE; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_NESTED_WITH_QUOTES; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_ONLINE; -import static org.hamcrest.Matchers.anyOf; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.core.StringStartsWith.startsWith; - -/** - * Tests to cover requests with "?format=csv" parameter - */ -public class CsvFormatResponseIT extends SQLIntegTestCase { - - private boolean flatOption = false; - - @Override - protected void init() throws Exception { - loadIndex(Index.ACCOUNT); - loadIndex(Index.NESTED); - loadIndex(Index.NESTED_WITH_QUOTES); - loadIndex(Index.DOG); - loadIndex(Index.GAME_OF_THRONES); - loadIndex(Index.ONLINE); - } - - @Override - protected Request getSqlRequest(String request, boolean explain) { - - Request sqlRequest = super.getSqlRequest(request, explain); - sqlRequest.addParameter("format", "csv"); - sqlRequest.addParameter("flat", flatOption ? "true" : "false"); - return sqlRequest; - } - - @Test - public void allPercentilesByDefault() throws IOException { - - final String query = String.format(Locale.ROOT, "SELECT PERCENTILES(age) FROM %s", TEST_INDEX_ACCOUNT); - final String result = executeQueryWithStringOutput(query); - - final String expectedHeaders = "PERCENTILES(age).1.0,PERCENTILES(age).5.0,PERCENTILES(age).25.0," + - "PERCENTILES(age).50.0,PERCENTILES(age).75.0,PERCENTILES(age).95.0,PERCENTILES(age).99.0"; - Assert.assertThat(result, containsString(expectedHeaders)); - } - - @Test - public void specificPercentilesIntAndDouble() throws IOException { - - final String query = String.format(Locale.ROOT, "SELECT PERCENTILES(age,10,49.0) FROM %s", - TEST_INDEX_ACCOUNT); - final String result = executeQueryWithStringOutput(query); - - final String[] unexpectedPercentiles = {"1.0", "5.0", "25.0", "50.0", "75.0", "95.0", "99.0"}; - final String expectedHeaders = "\"PERCENTILES(age,10,49.0).10.0\",\"PERCENTILES(age,10,49.0).49.0\""; - Assert.assertThat(result, containsString(expectedHeaders)); - for (final String unexpectedPercentile : unexpectedPercentiles) { - Assert.assertThat(result, not(containsString("PERCENTILES(age,10,49.0)." + unexpectedPercentile))); - } - } - - @Test - public void nestedObjectsAndArraysAreQuoted() throws IOException { - - final String query = String.format(Locale.ROOT, "SELECT * FROM %s WHERE _id = 5", - TEST_INDEX_NESTED_TYPE); - final String result = executeQueryWithStringOutput(query); - - final String expectedMyNum = "\"[3, 4]\""; - final String expectedComment = "\"{data=[aa, bb], likes=10}\""; - final String expectedMessage = "\"[{dayOfWeek=6, author=zz, info=zz}]\""; - - Assert.assertThat(result, containsString(expectedMyNum)); - Assert.assertThat(result, containsString(expectedComment)); - Assert.assertThat(result, containsString(expectedMessage)); - } - - @Test - public void arraysAreQuotedInFlatMode() throws IOException { - - setFlatOption(true); - - final String query = String.format(Locale.ROOT, "SELECT * FROM %s WHERE _id = 5", - TEST_INDEX_NESTED_TYPE); - final String result = executeQueryWithStringOutput(query); - - final String expectedMyNum = "\"[3, 4]\""; - final String expectedCommentData = "\"[aa, bb]\""; - final String expectedMessage = "\"[{dayOfWeek=6, author=zz, info=zz}]\""; - - Assert.assertThat(result, containsString(expectedMyNum)); - Assert.assertThat(result, containsString(expectedCommentData)); - Assert.assertThat(result, containsString(expectedMessage)); - - setFlatOption(false); - } - - @Test - public void doubleQuotesAreEscapedWithDoubleQuotes() throws IOException { - final String query = "SELECT * FROM " + TEST_INDEX_NESTED_WITH_QUOTES; - - final CSVResult csvResult = executeCsvRequest(query, false); - final List rows = csvResult.getLines(); - Assert.assertThat(rows.size(), equalTo(2)); - - final String expectedValue1 = "\"[{dayOfWeek=6, author=z\"\"z, info=zz}]\""; - final String expectedValue2 = "\"[{dayOfWeek=3, author=this \"\"value\"\" contains quotes, info=rr}]\""; - - for (String row : rows) { - Assert.assertThat(row, anyOf(containsString(expectedValue1), containsString(expectedValue2))); - } - } - - @Test - public void fieldOrder() throws IOException { - - final String[] expectedFields = {"age", "firstname", "address", "gender", "email"}; - - verifyFieldOrder(expectedFields); - } - - @Test - public void fieldOrderOther() throws IOException { - - final String[] expectedFields = {"email", "firstname", "age", "gender", "address"}; - - verifyFieldOrder(expectedFields); - } - - @Ignore("Getting parser error") - @Test - public void fieldOrderWithScriptFields() throws IOException { - - final String[] expectedFields = {"email", "script1", "script2", "gender", "address"}; - final String query = String.format(Locale.ROOT, "SELECT email, " + - "script(script1, \"doc['balance'].value * 2\"), " + - "script(script2, painless, \"doc['balance'].value + 10\"), gender, address " + - "FROM %s WHERE email='amberduke@pyrami.com'", TEST_INDEX_ACCOUNT); - - verifyFieldOrder(expectedFields, query); - } - - //region Tests migrated from CSVResultsExtractorTests - - @Test - public void simpleSearchResultNotNestedNotFlatNoAggs() throws Exception { - String query = String.format(Locale.ROOT, "select dog_name,age from %s order by age", TEST_INDEX_DOG); - final CSVResult csvResult = executeCsvRequest(query, false); - - List headers = csvResult.getHeaders(); - Assert.assertEquals(2, headers.size()); - Assert.assertTrue("name should be on headers", headers.contains("dog_name")); - Assert.assertTrue("age should be on headers", headers.contains("age")); - - List lines = csvResult.getLines(); - Assert.assertEquals(2, lines.size()); - Assert.assertTrue("rex,2".equals(lines.get(0)) || "2,rex".equals(lines.get(0))); - Assert.assertTrue("snoopy,4".equals(lines.get(1)) || "4,snoopy".equals(lines.get(1))); - } - - @Test - public void simpleSearchResultWithNestedNotFlatNoAggs() throws Exception { - String query = String.format(Locale.ROOT, "select name,house from %s", - TEST_INDEX_GAME_OF_THRONES); - CSVResult csvResult = executeCsvRequest(query, false); - - List headers = csvResult.getHeaders(); - Assert.assertEquals(2, headers.size()); - Assert.assertTrue("name should be on headers", headers.contains("name")); - Assert.assertTrue("house should be on headers", headers.contains("house")); - - List lines = csvResult.getLines(); - Assert.assertEquals(7, lines.size()); - - Assert.assertThat(lines, hasRow(null, "Targaryen", - Arrays.asList("firstname=Daenerys", "lastname=Targaryen", "ofHerName=1"), true)); - Assert.assertThat(lines, hasRow(null, "Stark", - Arrays.asList("firstname=Eddard", "lastname=Stark", "ofHisName=1"), true)); - Assert.assertThat(lines, hasRow(null, "Stark", - Arrays.asList("firstname=Brandon", "lastname=Stark", "ofHisName=4"), true)); - Assert.assertThat(lines, hasRow(null, "Lannister", - Arrays.asList("firstname=Jaime", "lastname=Lannister", "ofHisName=1"), true)); - } - - @Ignore("headers incorrect in case of nested fields") - @Test - public void simpleSearchResultWithNestedOneFieldNotFlatNoAggs() throws Exception { - String query = String.format(Locale.ROOT, "select name.firstname,house from %s", - TEST_INDEX_GAME_OF_THRONES); - CSVResult csvResult = executeCsvRequest(query, false); - - List headers = csvResult.getHeaders(); - Assert.assertEquals(2, headers.size()); - Assert.assertTrue("name should be on headers", headers.contains("name")); - Assert.assertTrue("house should be on headers", headers.contains("house")); - - List lines = csvResult.getLines(); - Assert.assertEquals(7, lines.size()); - Assert.assertThat(lines, hasItem("{firstname=Daenerys},Targaryen")); - Assert.assertThat(lines, hasItem("{firstname=Eddard},Stark")); - Assert.assertThat(lines, hasItem("{firstname=Brandon},Stark")); - Assert.assertThat(lines, hasItem("{firstname=Jaime},Lannister")); - - } - - @Ignore("headers incorrect in case of nested fields") - @Test - public void simpleSearchResultWithNestedTwoFieldsFromSameNestedNotFlatNoAggs() throws Exception { - String query = String.format(Locale.ROOT, "select name.firstname,name.lastname,house from %s", - TEST_INDEX_GAME_OF_THRONES); - CSVResult csvResult = executeCsvRequest(query, false); - - List headers = csvResult.getHeaders(); - Assert.assertEquals(2, headers.size()); - Assert.assertTrue("name should be on headers", headers.contains("name")); - Assert.assertTrue("house should be on headers", headers.contains("house")); - - List lines = csvResult.getLines(); - Assert.assertEquals(7, lines.size()); - - Assert.assertThat(lines, hasRow(null, "Targaryen", - Arrays.asList("firstname=Daenerys", "lastname=Targaryen"), true)); - Assert.assertThat(lines, hasRow(null, "Stark", - Arrays.asList("firstname=Eddard", "lastname=Stark"), true)); - Assert.assertThat(lines, hasRow(null, "Stark", - Arrays.asList("firstname=Brandon", "lastname=Stark"), true)); - Assert.assertThat(lines, hasRow(null, "Lannister", - Arrays.asList("firstname=Jaime", "lastname=Lannister"), true)); - } - - @Test - public void simpleSearchResultWithNestedWithFlatNoAggs() throws Exception { - String query = String.format(Locale.ROOT, "select name.firstname,house from %s", - TEST_INDEX_GAME_OF_THRONES); - CSVResult csvResult = executeCsvRequest(query, true); - - List headers = csvResult.getHeaders(); - Assert.assertEquals(2, headers.size()); - Assert.assertTrue("name.firstname should be on headers", headers.contains("name.firstname")); - Assert.assertTrue("house should be on headers", headers.contains("house")); - - List lines = csvResult.getLines(); - Assert.assertEquals(7, lines.size()); - Assert.assertTrue(lines.contains("Daenerys,Targaryen")); - Assert.assertTrue(lines.contains("Eddard,Stark")); - Assert.assertTrue(lines.contains("Brandon,Stark")); - Assert.assertTrue(lines.contains("Jaime,Lannister")); - } - - @Test - public void joinSearchResultNotNestedNotFlatNoAggs() throws Exception { - String query = String.format(Locale.ROOT, "select c.gender , h.hname,h.words from %s c " + - "JOIN %s h " + - "on h.hname = c.house ", TEST_INDEX_GAME_OF_THRONES, TEST_INDEX_GAME_OF_THRONES); - CSVResult csvResult = executeCsvRequest(query, false); - - List headers = csvResult.getHeaders(); - Assert.assertEquals(3, headers.size()); - Assert.assertTrue("c.gender should be on headers", headers.contains("c.gender")); - Assert.assertTrue("h.hname should be on headers", headers.contains("h.hname")); - Assert.assertTrue("h.words should be on headers", headers.contains("h.words")); - - List lines = csvResult.getLines(); - Assert.assertEquals(4, lines.size()); - - Assert.assertThat(lines, hasRow(null, null, Arrays.asList("F", "fireAndBlood", "Targaryen"), false)); - } - - @Test - public void simpleNumericValueAgg() throws Exception { - String query = String.format(Locale.ROOT, "select count(*) from %s ", TEST_INDEX_DOG); - CSVResult csvResult = executeCsvRequest(query, false); - - List headers = csvResult.getHeaders(); - Assert.assertEquals(1, headers.size()); - Assert.assertEquals("COUNT(*)", headers.get(0)); - - - List lines = csvResult.getLines(); - Assert.assertEquals(1, lines.size()); - Assert.assertEquals("2.0", lines.get(0)); - - } - - @Test - public void simpleNumericValueAggWithAlias() throws Exception { - String query = String.format(Locale.ROOT, "select avg(age) as myAlias from %s ", TEST_INDEX_DOG); - CSVResult csvResult = executeCsvRequest(query, false); - - List headers = csvResult.getHeaders(); - Assert.assertEquals(1, headers.size()); - Assert.assertEquals("myAlias", headers.get(0)); - - - List lines = csvResult.getLines(); - Assert.assertEquals(1, lines.size()); - Assert.assertEquals("3.0", lines.get(0)); - - } - - @Test - public void twoNumericAggWithAlias() throws Exception { - String query = String.format(Locale.ROOT, "select count(*) as count, avg(age) as myAlias from %s ", - TEST_INDEX_DOG); - CSVResult csvResult = executeCsvRequest(query, false); - - List headers = csvResult.getHeaders(); - Assert.assertEquals(2, headers.size()); - - Assert.assertTrue(headers.contains("count")); - Assert.assertTrue(headers.contains("myAlias")); - - - List lines = csvResult.getLines(); - Assert.assertEquals(1, lines.size()); - if (headers.get(0).equals("count")) { - Assert.assertEquals("2.0,3.0", lines.get(0)); - } else { - Assert.assertEquals("3.0,2.0", lines.get(0)); - } - - } - - @Test - public void aggAfterTermsGroupBy() throws Exception { - String query = String.format(Locale.ROOT, "SELECT COUNT(*) FROM %s GROUP BY gender", - TEST_INDEX_ACCOUNT); - CSVResult csvResult = executeCsvRequest(query, false); - List headers = csvResult.getHeaders(); - Assert.assertEquals(1, headers.size()); - assertThat(headers, contains(equalTo("COUNT(*)"))); - - List lines = csvResult.getLines(); - Assert.assertEquals(2, lines.size()); - assertThat(lines, containsInAnyOrder(equalTo("507.0"), equalTo("493.0"))); - } - - @Test - public void aggAfterTwoTermsGroupBy() throws Exception { - String query = String.format(Locale.ROOT, - "SELECT COUNT(*) FROM %s where age in (35,36) GROUP BY gender,age", - TEST_INDEX_ACCOUNT); - CSVResult csvResult = executeCsvRequest(query, false); - List headers = csvResult.getHeaders(); - Assert.assertEquals(1, headers.size()); - assertThat(headers, contains(equalTo("COUNT(*)"))); - - List lines = csvResult.getLines(); - Assert.assertEquals(4, lines.size()); - assertThat(lines, containsInAnyOrder( - equalTo("31.0"), - equalTo("28.0"), - equalTo("21.0"), - equalTo("24.0"))); - } - - @Test - public void multipleAggAfterTwoTermsGroupBy() throws Exception { - String query = String.format(Locale.ROOT, - "SELECT COUNT(*) , sum(balance) FROM %s where age in (35,36) GROUP BY gender,age", - TEST_INDEX_ACCOUNT); - CSVResult csvResult = executeCsvRequest(query, false); - List headers = csvResult.getHeaders(); - Assert.assertEquals(2, headers.size()); - assertThat(headers, contains(equalTo("COUNT(*)"), equalTo("SUM(balance)"))); - - List lines = csvResult.getLines(); - Assert.assertEquals(4, lines.size()); - assertThat(lines, containsInAnyOrder( - equalTo("31.0,647425.0"), - equalTo("28.0,678337.0"), - equalTo("21.0,505660.0"), - equalTo("24.0,472771.0"))); - } - - @Test - public void dateHistogramTest() throws Exception { - String query = String.format(Locale.ROOT, "select count(*) from %s" + - " group by date_histogram('field'='insert_time','fixed_interval'='4d','alias'='days')", TEST_INDEX_ONLINE); - CSVResult csvResult = executeCsvRequest(query, false); - List headers = csvResult.getHeaders(); - Assert.assertEquals(1, headers.size()); - assertThat(headers, contains(equalTo("COUNT(*)"))); - - List lines = csvResult.getLines(); - Assert.assertEquals(3, lines.size()); - assertThat(lines, containsInAnyOrder( - equalTo("477.0"), - equalTo("5664.0"), - equalTo("3795.0"))); - } - - @Test - public void statsAggregationTest() throws Exception { - String query = String.format(Locale.ROOT, "SELECT STATS(age) FROM %s", TEST_INDEX_ACCOUNT); - CSVResult csvResult = executeCsvRequest(query, false); - List headers = csvResult.getHeaders(); - Assert.assertEquals(5, headers.size()); - Assert.assertEquals("STATS(age).count", headers.get(0)); - Assert.assertEquals("STATS(age).sum", headers.get(1)); - Assert.assertEquals("STATS(age).avg", headers.get(2)); - Assert.assertEquals("STATS(age).min", headers.get(3)); - Assert.assertEquals("STATS(age).max", headers.get(4)); - - List lines = csvResult.getLines(); - Assert.assertEquals(1, lines.size()); - Assert.assertEquals("1000,30171.0,30.171,20.0,40.0", lines.get(0)); - } - - @Test - public void extendedStatsAggregationTest() throws Exception { - String query = String.format(Locale.ROOT, "SELECT EXTENDED_STATS(age) FROM %s", TEST_INDEX_ACCOUNT); - CSVResult csvResult = executeCsvRequest(query, false); - List headers = csvResult.getHeaders(); - - final String[] expectedHeaders = {"EXTENDED_STATS(age).count", "EXTENDED_STATS(age).sum", - "EXTENDED_STATS(age).avg", "EXTENDED_STATS(age).min", "EXTENDED_STATS(age).max", - "EXTENDED_STATS(age).sumOfSquares", "EXTENDED_STATS(age).variance", "EXTENDED_STATS(age).stdDeviation"}; - - Assert.assertEquals(expectedHeaders.length, headers.size()); - Assert.assertThat(headers, contains(expectedHeaders)); - - List lines = csvResult.getLines(); - Assert.assertEquals(1, lines.size()); - String line = lines.get(0); - Assert.assertTrue(line.startsWith("1000,30171.0,30.171,20.0,40.0,946393.0")); - Assert.assertTrue(line.contains(",6.008")); - Assert.assertTrue(line.contains(",36.103")); - } - - @Test - public void percentileAggregationTest() throws Exception { - String query = String.format(Locale.ROOT, "select percentiles(age) as per from %s where age > 31", - TEST_INDEX_ACCOUNT); - CSVResult csvResult = executeCsvRequest(query, false); - List headers = csvResult.getHeaders(); - Assert.assertEquals(7, headers.size()); - Assert.assertEquals("per.1.0", headers.get(0)); - Assert.assertEquals("per.5.0", headers.get(1)); - Assert.assertEquals("per.25.0", headers.get(2)); - Assert.assertEquals("per.50.0", headers.get(3)); - Assert.assertEquals("per.75.0", headers.get(4)); - Assert.assertEquals("per.95.0", headers.get(5)); - Assert.assertEquals("per.99.0", headers.get(6)); - - - List lines = csvResult.getLines(); - Assert.assertEquals(1, lines.size()); - Assert.assertEquals("32.0,32.0,34.0,36.0,38.0,40.0,40.0", lines.get(0)); - } - - @Test - public void includeTypeAndNotScore() throws Exception { - String query = String.format(Locale.ROOT, "select age , firstname from %s where age > 31 limit 2", - TEST_INDEX_ACCOUNT); - CSVResult csvResult = executeCsvRequest(query, false, false, true, false); - List headers = csvResult.getHeaders(); - Assert.assertEquals(3, headers.size()); - Assert.assertTrue(headers.contains("age")); - Assert.assertTrue(headers.contains("firstname")); - Assert.assertTrue(headers.contains("_type")); - List lines = csvResult.getLines(); - Assert.assertTrue(lines.get(0).contains(",_doc") || lines.get(0).contains("_doc,")); - Assert.assertTrue(lines.get(1).contains(",_doc") || lines.get(1).contains("_doc,")); - } - - @Test - public void includeScoreAndNotType() throws Exception { - String query = String.format(Locale.ROOT, - "select age , firstname from %s where age > 31 order by _score desc limit 2 ", - TEST_INDEX_ACCOUNT); - CSVResult csvResult = executeCsvRequest(query, false, true, false, false); - List headers = csvResult.getHeaders(); - Assert.assertEquals(3, headers.size()); - Assert.assertTrue(headers.contains("age")); - Assert.assertTrue(headers.contains("firstname")); - Assert.assertTrue(headers.contains("_score")); - List lines = csvResult.getLines(); - Assert.assertTrue(lines.get(0).contains("1.0")); - Assert.assertTrue(lines.get(1).contains("1.0")); - } - - @Test - public void includeScoreAndType() throws Exception { - String query = String.format(Locale.ROOT, - "select age , firstname from %s where age > 31 order by _score desc limit 2 ", - TEST_INDEX_ACCOUNT); - CSVResult csvResult = executeCsvRequest(query, false, true, true, false); - List headers = csvResult.getHeaders(); - Assert.assertEquals(4, headers.size()); - Assert.assertTrue(headers.contains("age")); - Assert.assertTrue(headers.contains("firstname")); - Assert.assertTrue(headers.contains("_score")); - Assert.assertTrue(headers.contains("_type")); - List lines = csvResult.getLines(); - String firstLine = lines.get(0); - Assert.assertTrue(firstLine.contains("_doc,1.0") || firstLine.contains("1.0,_doc")); - Assert.assertTrue(lines.get(1).contains("_doc,1.0") || lines.get(1).contains("1.0,_doc")); - } - - /* todo: more tests: - * filter/nested and than metric - * histogram - * geo - */ - - @Test - public void scriptedField() throws Exception { - String query = String.format(Locale.ROOT, - "select age+1 as agePlusOne ,age , firstname from %s where age = 31 limit 1", - TEST_INDEX_ACCOUNT); - CSVResult csvResult = executeCsvRequest(query, false); - List headers = csvResult.getHeaders(); - Assert.assertEquals(3, headers.size()); - Assert.assertTrue(headers.contains("agePlusOne")); - Assert.assertTrue(headers.contains("age")); - Assert.assertTrue(headers.contains("firstname")); - List lines = csvResult.getLines(); - Assert.assertTrue(lines.get(0).contains("32,31") || lines.get(0).contains("32.0,31.0") || - lines.get(0).contains("31,32") || lines.get(0).contains("31.0,32.0")); - } - - @Ignore("separator not exposed") - @Test - public void twoCharsSeperator() throws Exception { - String query = String.format(Locale.ROOT, "select dog_name,age from %s order by age", TEST_INDEX_DOG); - CSVResult csvResult = executeCsvRequest(query, false); - - List headers = csvResult.getHeaders(); - Assert.assertEquals(2, headers.size()); - Assert.assertTrue("name should be on headers", headers.contains("dog_name")); - Assert.assertTrue("age should be on headers", headers.contains("age")); - - List lines = csvResult.getLines(); - Assert.assertEquals(2, lines.size()); - Assert.assertTrue("rex||2".equals(lines.get(0)) || "2||rex".equals(lines.get(0))); - Assert.assertTrue("snoopy||4".equals(lines.get(1)) || "4||snoopy".equals(lines.get(1))); - - } - - @Test - public void includeIdAndNotTypeOrScore() throws Exception { - String query = String.format(Locale.ROOT, - "select age , firstname from %s where lastname = 'Marquez' ", TEST_INDEX_ACCOUNT); - CSVResult csvResult = executeCsvRequest(query, false, false, false, true); - List headers = csvResult.getHeaders(); - Assert.assertEquals(3, headers.size()); - Assert.assertTrue(headers.contains("age")); - Assert.assertTrue(headers.contains("firstname")); - Assert.assertTrue(headers.contains("_id")); - List lines = csvResult.getLines(); - Assert.assertTrue(lines.get(0).contains(",437") || lines.get(0).contains("437,")); - } - - @Test - public void includeIdAndTypeButNoScore() throws Exception { - String query = String.format(Locale.ROOT, - "select age , firstname from %s where lastname = 'Marquez' ", TEST_INDEX_ACCOUNT); - CSVResult csvResult = executeCsvRequest(query, false, false, true, true); - List headers = csvResult.getHeaders(); - Assert.assertEquals(4, headers.size()); - Assert.assertTrue(headers.contains("age")); - Assert.assertTrue(headers.contains("firstname")); - Assert.assertTrue(headers.contains("_id")); - Assert.assertTrue(headers.contains("_type")); - List lines = csvResult.getLines(); - Assert.assertTrue(lines.get(0).contains("_doc,437") || lines.get(0).contains("437,_doc")); - } - //endregion Tests migrated from CSVResultsExtractorTests - - @Test - public void sensitiveCharacterSanitizeTest() throws IOException { - String requestBody = - "{" + - " \"=cmd|' /C notepad'!_xlbgnm.A1\": \"+cmd|' /C notepad'!_xlbgnm.A1\",\n" + - " \"-cmd|' /C notepad'!_xlbgnm.A1\": \"@cmd|' /C notepad'!_xlbgnm.A1\"\n" + - "}"; - - Request request = new Request("PUT", "/userdata/_doc/1?refresh=true"); - request.setJsonEntity(requestBody); - TestUtils.performRequest(client(), request); - - CSVResult csvResult = executeCsvRequest("SELECT * FROM userdata", false, false, false, false); - List headers = csvResult.getHeaders(); - Assert.assertEquals(2, headers.size()); - Assert.assertTrue(headers.contains("'=cmd|' /C notepad'!_xlbgnm.A1")); - Assert.assertTrue(headers.contains("'-cmd|' /C notepad'!_xlbgnm.A1")); - - List lines = csvResult.getLines(); - Assert.assertEquals(1, lines.size()); - Assert.assertTrue(lines.get(0).contains("'+cmd|' /C notepad'!_xlbgnm.A1")); - Assert.assertTrue(lines.get(0).contains("'@cmd|' /C notepad'!_xlbgnm.A1")); - } - - @Test - public void sensitiveCharacterSanitizeAndQuotedTest() throws IOException { - String requestBody = - "{" + - " \"=cmd|' /C notepad'!_xlbgnm.A1,,\": \",+cmd|' /C notepad'!_xlbgnm.A1\",\n" + - " \",@cmd|' /C notepad'!_xlbgnm.A1\": \"+cmd|' /C notepad,,'!_xlbgnm.A1\",\n" + - " \"-cmd|' /C notepad,,'!_xlbgnm.A1\": \",,,@cmd|' /C notepad'!_xlbgnm.A1\"\n" + - "}"; - - Request request = new Request("PUT", "/userdata2/_doc/1?refresh=true"); - request.setJsonEntity(requestBody); - TestUtils.performRequest(client(), request); - - CSVResult csvResult = executeCsvRequest("SELECT * FROM userdata2", false, false, false, false); - String headers = String.join(",", csvResult.getHeaders()); - Assert.assertTrue(headers.contains("\"'=cmd|' /C notepad'!_xlbgnm.A1,,\"")); - Assert.assertTrue(headers.contains("\",@cmd|' /C notepad'!_xlbgnm.A1\"")); - Assert.assertTrue(headers.contains("\"'-cmd|' /C notepad,,'!_xlbgnm.A1\"")); - - List lines = csvResult.getLines(); - Assert.assertEquals(1, lines.size()); - Assert.assertTrue(lines.get(0).contains("\",+cmd|' /C notepad'!_xlbgnm.A1\"")); - Assert.assertTrue(lines.get(0).contains("\"'+cmd|' /C notepad,,'!_xlbgnm.A1\"")); - Assert.assertTrue(lines.get(0).contains("\",,,@cmd|' /C notepad'!_xlbgnm.A1\"")); - } - - @Test - public void selectFunctionAsFieldTest() throws IOException { - String query = "select log(age) from " + TEST_INDEX_ACCOUNT; - CSVResult result = executeCsvRequest(query, false, false, false, false); - List headers = result.getHeaders(); - Assert.assertEquals(1, headers.size()); - } - - private void verifyFieldOrder(final String[] expectedFields) throws IOException { - - final String fields = String.join(", ", expectedFields); - final String query = String.format(Locale.ROOT, "SELECT %s FROM %s " + - "WHERE email='amberduke@pyrami.com'", fields, TEST_INDEX_ACCOUNT); - - verifyFieldOrder(expectedFields, query); - } - - private void verifyFieldOrder(final String[] expectedFields, final String query) throws IOException { - - final String result = executeQueryWithStringOutput(query); - - final String expectedHeader = String.join(",", expectedFields); - Assert.assertThat(result, startsWith(expectedHeader)); - } - - private void setFlatOption(boolean flat) { - - this.flatOption = flat; - } - - private CSVResult executeCsvRequest(final String query, boolean flat) throws IOException { - - return executeCsvRequest(query, flat, false, false, false); - } - - private CSVResult executeCsvRequest(final String query, boolean flat, boolean includeScore, - boolean includeType, boolean includeId) throws IOException { - - final String requestBody = super.makeRequest(query); - final String endpoint = String.format(Locale.ROOT, - "/_opendistro/_sql?format=csv&flat=%b&_id=%b&_score=%b&_type=%b", - flat, includeId, includeScore, includeType); - final Request sqlRequest = new Request("POST", endpoint); - sqlRequest.setJsonEntity(requestBody); - RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); - restOptionsBuilder.addHeader("Content-Type", "application/json"); - sqlRequest.setOptions(restOptionsBuilder); - - final Response response = client().performRequest(sqlRequest); - Assert.assertEquals(200, response.getStatusLine().getStatusCode()); - final String responseString = TestUtils.getResponseBody(response, true); - - return csvResultFromStringResponse(responseString); - } - - private CSVResult csvResultFromStringResponse(final String response) { - - final List rows = new ArrayList<>(); - - final String newLine = String.format(Locale.ROOT, "%n"); - int newLineIndex = response.indexOf(newLine); - - final String headerLine; - - if (-1 == newLineIndex) { - // assuming only headers - headerLine = response.trim(); - } else { - headerLine = response.substring(0, newLineIndex).trim(); - final String rowLines = response.substring(newLineIndex + newLine.length()).trim(); - if (!rowLines.isEmpty()) { - rows.addAll(Arrays.asList(rowLines.split(newLine))); - } - } - - final List headers = new ArrayList<>(Arrays.asList(headerLine.split(","))); - return new CSVResult(headers, rows); - } - - private static AnyOf> hasRow(final String prefix, final String suffix, final List items, - final boolean areItemsNested) { - - final Collection> permutations = TestUtils.getPermutations(items); - - final List>> matchers = permutations.stream().map(permutation -> { - - final String delimiter = areItemsNested ? ", " : ","; - final String objectField = String.join(delimiter, permutation); - final String row = String.format(Locale.ROOT, "%s%s%s%s%s", - printablePrefix(prefix), areItemsNested ? "\"{" : "", - objectField, areItemsNested ? "}\"" : "", printableSuffix(suffix)); - return hasItem(row); - - }).collect(Collectors.toCollection(LinkedList::new)); - - return anyOf(matchers); - } - - private static String printablePrefix(final String prefix) { - - if (prefix == null || prefix.trim().isEmpty()) { - return ""; - } - - return prefix + ","; - } - - private static String printableSuffix(final String suffix) { - - if (suffix == null || suffix.trim().isEmpty()) { - return ""; - } - - return "," + suffix; - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/CursorIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/CursorIT.java deleted file mode 100644 index c2d3733202..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/CursorIT.java +++ /dev/null @@ -1,484 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; -import org.json.JSONArray; -import org.json.JSONObject; -import org.junit.Test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getResponseBody; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_ACCOUNT; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_DATE_TIME; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_NESTED_SIMPLE; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.containsString; - -public class CursorIT extends SQLIntegTestCase { - - private static final String CURSOR = "cursor"; - private static final String DATAROWS = "datarows"; - private static final String SCHEMA = "schema"; - private static final String JDBC = "jdbc"; - private static final String NEW_LINE = "\n"; - - @Override - protected void init() throws Exception { - loadIndex(Index.ACCOUNT); - enableCursorClusterSetting(); - } - - /** - * Acceptable fetch_size are positive numbers. - * For example 0, 24, 53.0, "110" (parsable string) , "786.23" - * Negative values should throw 400 - */ - @Test - public void invalidNegativeFetchSize() throws IOException { - String query = StringUtils.format("SELECT firstname, state FROM %s", TestsConstants.TEST_INDEX_ACCOUNT); - Response response = null; - try { - String queryResult = executeFetchQuery(query, -2, JDBC); - } catch (ResponseException ex) { - response = ex.getResponse(); - } - - JSONObject resp = new JSONObject(TestUtils.getResponseBody(response)); - assertThat(resp.getInt("status"), equalTo(400)); - assertThat(resp.query("/error/reason"), equalTo("Invalid SQL query")); - assertThat(resp.query("/error/details"), equalTo("Fetch_size must be greater or equal to 0")); - assertThat(resp.query("/error/type"), equalTo("IllegalArgumentException")); - } - - /** - * Non-numeric fetch_size value should throw 400 - */ - @Test - public void invalidNonNumericFetchSize() throws IOException { - String query = StringUtils.format("SELECT firstname, state FROM %s", TestsConstants.TEST_INDEX_ACCOUNT); - Response response = null; - try { - String queryResult = executeFetchAsStringQuery(query, "hello world", JDBC); - } catch (ResponseException ex) { - response = ex.getResponse(); - } - - JSONObject resp = new JSONObject(TestUtils.getResponseBody(response)); - assertThat(resp.getInt("status"), equalTo(400)); - assertThat(resp.query("/error/reason"), equalTo("Invalid SQL query")); - assertThat(resp.query("/error/details"), equalTo("Failed to parse field [fetch_size]")); - assertThat(resp.query("/error/type"), equalTo("IllegalArgumentException")); - } - - @Test - public void testExceptionOnCursorExplain() throws IOException { - String cursorRequest = "{\"cursor\":\"d:eyJhIjp7fSwicyI6IkRYRjFaWEo1\"}"; - Request sqlRequest = getSqlRequest(cursorRequest, true); - Response response = null; - try { - String queryResult = executeRequest(sqlRequest); - } catch (ResponseException ex) { - response = ex.getResponse(); - } - - JSONObject resp = new JSONObject(TestUtils.getResponseBody(response)); - assertThat(resp.getInt("status"), equalTo(400)); - assertThat(resp.query("/error/reason"), equalTo("Invalid SQL query")); - assertThat(resp.query("/error/details"), equalTo("Invalid request. Cannot explain cursor")); - assertThat(resp.query("/error/type"), equalTo("IllegalArgumentException")); - } - - /** - * For fetch_size = 0, default to non-pagination behaviour for simple queries - * This can be verified by checking that cursor is not present, and old default limit applies - */ - @Test - public void noPaginationWhenFetchSizeZero() throws IOException { - String selectQuery = StringUtils.format("SELECT firstname, state FROM %s", TEST_INDEX_ACCOUNT); - JSONObject response = new JSONObject(executeFetchQuery(selectQuery, 0, JDBC)); - assertFalse(response.has(CURSOR)); - assertThat(response.getJSONArray(DATAROWS).length(), equalTo(200)); - } - - /** - * The index has 1000 records, with fetch size of 50 we should get 20 pages with no cursor on last page - */ - @Test - public void validNumberOfPages() throws IOException { - String selectQuery = StringUtils.format("SELECT firstname, state FROM %s", TEST_INDEX_ACCOUNT); - JSONObject response = new JSONObject(executeFetchQuery(selectQuery, 50, JDBC)); - String cursor = response.getString(CURSOR); - int pageCount = 1; - - while (!cursor.isEmpty()) { //this condition also checks that there is no cursor on last page - response = executeCursorQuery(cursor); - cursor = response.optString(CURSOR); - pageCount++; - } - - assertThat(pageCount, equalTo(20)); - - // using random value here, with fetch size of 28 we should get 36 pages (ceil of 1000/28) - response = new JSONObject(executeFetchQuery(selectQuery, 28, JDBC)); - cursor = response.getString(CURSOR); - System.out.println(response); - pageCount = 1; - - while (!cursor.isEmpty()) { - response = executeCursorQuery(cursor); - cursor = response.optString(CURSOR); - pageCount++; - } - assertThat(pageCount, equalTo(36)); - } - - - @Test - public void validTotalResultWithAndWithoutPagination() throws IOException { - // simple query - accounts index has 1000 docs, using higher limit to get all docs - String selectQuery = StringUtils.format("SELECT firstname, state FROM %s ", TEST_INDEX_ACCOUNT ); - verifyWithAndWithoutPaginationResponse(selectQuery + " LIMIT 2000" , selectQuery , 80); - } - - @Test - public void validTotalResultWithAndWithoutPaginationWhereClause() throws IOException { - String selectQuery = StringUtils.format( - "SELECT firstname, state FROM %s WHERE balance < 25000 AND age > 32", TEST_INDEX_ACCOUNT - ); - verifyWithAndWithoutPaginationResponse(selectQuery + " LIMIT 2000" , selectQuery , 17); - } - - @Test - public void validTotalResultWithAndWithoutPaginationOrderBy() throws IOException { - String selectQuery = StringUtils.format( - "SELECT firstname, state FROM %s ORDER BY balance DESC ", TEST_INDEX_ACCOUNT - ); - verifyWithAndWithoutPaginationResponse(selectQuery + " LIMIT 2000" , selectQuery , 26); - } - - @Test - public void validTotalResultWithAndWithoutPaginationWhereAndOrderBy() throws IOException { - String selectQuery = StringUtils.format( - "SELECT firstname, state FROM %s WHERE balance < 25000 ORDER BY balance ASC ", TEST_INDEX_ACCOUNT - ); - verifyWithAndWithoutPaginationResponse(selectQuery + " LIMIT 2000" , selectQuery , 80); - - } - - @Test - public void validTotalResultWithAndWithoutPaginationNested() throws IOException { - loadIndex(Index.NESTED_SIMPLE); - String selectQuery = StringUtils.format( - "SELECT name, a.city, a.state FROM %s m , m.address as a ", TEST_INDEX_NESTED_SIMPLE - ); - verifyWithAndWithoutPaginationResponse(selectQuery + " LIMIT 2000" , selectQuery , 1); - } - - @Test - public void noCursorWhenResultsLessThanFetchSize() throws IOException { - // fetch_size is 100, but actual number of rows returned from ElasticSearch is 97 - // a scroll context will be opened but will be closed after first page as all records are fetched - String selectQuery = StringUtils.format( - "SELECT * FROM %s WHERE balance < 25000 AND age > 36 LIMIT 2000", TEST_INDEX_ACCOUNT - ); - JSONObject response = new JSONObject(executeFetchQuery(selectQuery, 100, JDBC)); - assertFalse(response.has(CURSOR)); - } - - @Test - public void testCursorWithPreparedStatement() throws IOException { - JSONObject response = executeJDBCRequest(String.format("{" + - " \"fetch_size\": 200," + - " \"query\": \" SELECT age, state FROM %s WHERE age > ? OR state IN (?, ?)\"," + - " \"parameters\": [" + - " {" + - " \"type\": \"integer\"," + - " \"value\": 25" + - " }," + - " {" + - " \"type\": \"string\"," + - " \"value\": \"WA\"" + - " }," + - " {" + - " \"type\": \"string\"," + - " \"value\": \"UT\"" + - " }" + - " ]" + - "}", TestsConstants.TEST_INDEX_ACCOUNT)); - - assertTrue(response.has(CURSOR)); - } - - @Test - public void testRegressionOnDateFormatChange() throws IOException { - loadIndex(Index.DATETIME); - /** - * With pagination, the field should be date formatted to MySQL format as in - * @see PR #367 actualDateList = new ArrayList<>(); - String selectQuery = StringUtils.format("SELECT login_time FROM %s LIMIT 500", TEST_INDEX_DATE_TIME); - JSONObject response = new JSONObject(executeFetchQuery(selectQuery, 1, JDBC)); - String cursor = response.getString(CURSOR); - actualDateList.add(response.getJSONArray(DATAROWS).getJSONArray(0).getString(0)); - - while (!cursor.isEmpty()) { - response = executeCursorQuery(cursor); - cursor = response.optString(CURSOR); - actualDateList.add(response.getJSONArray(DATAROWS).getJSONArray(0).getString(0)); - } - - List expectedDateList = Arrays.asList( - "2015-01-01 00:00:00.000", - "2015-01-01 12:10:30.000", - "1585882955", // by existing design, this is not formatted in MySQL standard format - "2020-04-08 06:10:30.000"); - - assertThat(actualDateList, equalTo(expectedDateList)); - } - - - @Test - public void defaultBehaviorWhenCursorSettingIsDisabled() throws IOException { - updateClusterSettings(new ClusterSetting(PERSISTENT, "opendistro.sql.cursor.enabled", "false")); - String query = StringUtils.format("SELECT firstname, email, state FROM %s", TEST_INDEX_ACCOUNT); - JSONObject response = new JSONObject(executeFetchQuery(query, 100, JDBC)); - assertFalse(response.has(CURSOR)); - - updateClusterSettings(new ClusterSetting(PERSISTENT, "opendistro.sql.cursor.enabled", "true")); - query = StringUtils.format("SELECT firstname, email, state FROM %s", TEST_INDEX_ACCOUNT); - response = new JSONObject(executeFetchQuery(query, 100, JDBC)); - assertTrue(response.has(CURSOR)); - - wipeAllClusterSettings(); - } - - - @Test - public void testCursorSettings() throws IOException { - // reverting enableCursorClusterSetting() in init() method before checking defaults - updateClusterSettings(new ClusterSetting(PERSISTENT, "opendistro.sql.cursor.enabled", null)); - - // Assert default cursor settings - JSONObject clusterSettings = getAllClusterSettings(); - assertThat(clusterSettings.query("/defaults/opendistro.sql.cursor.enabled"), equalTo("false")); - assertThat(clusterSettings.query("/defaults/opendistro.sql.cursor.fetch_size"), equalTo("1000")); - assertThat(clusterSettings.query("/defaults/opendistro.sql.cursor.keep_alive"), equalTo("1m")); - - updateClusterSettings(new ClusterSetting(PERSISTENT, "opendistro.sql.cursor.enabled", "true")); - updateClusterSettings(new ClusterSetting(TRANSIENT, "opendistro.sql.cursor.fetch_size", "400")); - updateClusterSettings(new ClusterSetting(PERSISTENT, "opendistro.sql.cursor.keep_alive", "200s")); - - clusterSettings = getAllClusterSettings(); - assertThat(clusterSettings.query("/persistent/opendistro.sql.cursor.enabled"), equalTo("true")); - assertThat(clusterSettings.query("/transient/opendistro.sql.cursor.fetch_size"), equalTo("400")); - assertThat(clusterSettings.query("/persistent/opendistro.sql.cursor.keep_alive"), equalTo("200s")); - - wipeAllClusterSettings(); - } - - - @Test - public void testDefaultFetchSizeFromClusterSettings() throws IOException { - // the default fetch size is 1000 - // using non-nested query here as page will have more rows on flattening - String query = StringUtils.format("SELECT firstname, email, state FROM %s", TEST_INDEX_ACCOUNT); - JSONObject response = new JSONObject(executeFetchLessQuery(query, JDBC)); - JSONArray datawRows = response.optJSONArray(DATAROWS); - assertThat(datawRows.length(), equalTo(1000)); - - updateClusterSettings(new ClusterSetting(TRANSIENT, "opendistro.sql.cursor.fetch_size", "786")); - response = new JSONObject(executeFetchLessQuery(query, JDBC)); - datawRows = response.optJSONArray(DATAROWS); - assertThat(datawRows.length(), equalTo(786)); - assertTrue(response.has(CURSOR)); - - wipeAllClusterSettings(); - } - - @Test - public void testCursorCloseAPI() throws IOException { - // multiple invocation of closing cursor should return success - // fetch page using old cursor should throw error - String selectQuery = StringUtils.format( - "SELECT firstname, state FROM %s WHERE balance > 100 and age < 40", TEST_INDEX_ACCOUNT); - JSONObject result = new JSONObject(executeFetchQuery(selectQuery, 50, JDBC)); - String cursor = result.getString(CURSOR); - - // Retrieving next 10 pages out of remaining 19 pages - for(int i =0 ; i < 10 ; i++) { - result = executeCursorQuery(cursor); - cursor = result.optString(CURSOR); - } - //Closing the cursor - JSONObject closeResp = executeCursorCloseQuery(cursor); - assertThat(closeResp.getBoolean("succeeded"), equalTo(true)); - - //Closing the cursor multiple times is idempotent - for(int i =0 ; i < 5 ; i++) { - closeResp = executeCursorCloseQuery(cursor); - assertThat(closeResp.getBoolean("succeeded"), equalTo(true)); - } - - // using the cursor after its cleared, will throw exception - Response response = null; - try { - JSONObject queryResult = executeCursorQuery(cursor); - } catch (ResponseException ex) { - response = ex.getResponse(); - } - - JSONObject resp = new JSONObject(TestUtils.getResponseBody(response)); - assertThat(resp.getInt("status"), equalTo(404)); - assertThat(resp.query("/error/reason"), equalTo("all shards failed")); - assertThat(resp.query("/error/caused_by/reason").toString(), containsString("No search context found")); - assertThat(resp.query("/error/type"), equalTo("search_phase_execution_exception")); - } - - - @Test - public void invalidCursorIdNotDecodable() throws IOException { - // could be either not decode-able - String randomCursor = "d:eyJzY2hlbWEiOlt7Im5hbWUiOiJmaXJzdG5hbWUiLCJ0eXBlIjoidGV4dCJ9LHsibmFtZSI6InN0Y"; - - Response response = null; - try { - JSONObject resp = executeCursorQuery(randomCursor); - } catch (ResponseException ex) { - response = ex.getResponse(); - } - - JSONObject resp = new JSONObject(TestUtils.getResponseBody(response)); - assertThat(resp.getInt("status"), equalTo(400)); - assertThat(resp.query("/error/type"), equalTo("illegal_argument_exception")); - } - - /** - * The index has 1000 records, with fetch size of 50 and LIMIT in place - * we should get Math.ceil(limit/fetchSize) pages and LIMIT number of rows. - * Basically it should not retrieve all records in presence of a smaller LIMIT value. - */ - @Test - public void respectLimitPassedInSelectClause() throws IOException { - int limit = 234; - String selectQuery = StringUtils.format("SELECT age, balance FROM %s LIMIT %s", TEST_INDEX_ACCOUNT, limit); - JSONObject response = new JSONObject(executeFetchQuery(selectQuery, 50, JDBC)); - String cursor = response.getString(CURSOR); - int actualDataRowCount = response.getJSONArray(DATAROWS).length(); - int pageCount = 1; - - while (!cursor.isEmpty()) { - response = executeCursorQuery(cursor); - cursor = response.optString(CURSOR); - actualDataRowCount += response.getJSONArray(DATAROWS).length(); - pageCount++; - } - - assertThat(pageCount, equalTo(5)); - assertThat(actualDataRowCount, equalTo(limit)); - } - - - @Test - public void noPaginationWithNonJDBCFormat() throws IOException { - // checking for CSV, RAW format - String query = StringUtils.format("SELECT firstname, email, state FROM %s LIMIT 2000", TEST_INDEX_ACCOUNT); - String csvResult = executeFetchQuery(query, 100, "csv"); - String[] rows = csvResult.split(NEW_LINE); - // all the 1001 records (+1 for header) are retrieved instead of fetch_size number of records - assertThat(rows.length, equalTo(1001)); - - String rawResult = executeFetchQuery(query, 100, "raw"); - rows = rawResult.split(NEW_LINE); - // all the 1000 records (NO headers) are retrieved instead of fetch_size number of records - assertThat(rows.length, equalTo(1000)); - } - - - public void verifyWithAndWithoutPaginationResponse(String sqlQuery, String cursorQuery, int fetch_size) throws IOException { - // we are only checking here for schema and datarows - JSONObject withoutCursorResponse = new JSONObject(executeFetchQuery(sqlQuery, 0, JDBC)); - - JSONObject withCursorResponse = new JSONObject("{\"schema\":[],\"datarows\":[]}"); - JSONArray schema = withCursorResponse.getJSONArray(SCHEMA); - JSONArray dataRows = withCursorResponse.getJSONArray(DATAROWS); - - JSONObject response = new JSONObject(executeFetchQuery(cursorQuery, fetch_size, JDBC)); - response.optJSONArray(SCHEMA).forEach(schema::put); - response.optJSONArray(DATAROWS).forEach(dataRows::put); - - String cursor = response.getString(CURSOR); - while (!cursor.isEmpty()) { - response = executeCursorQuery(cursor); - response.optJSONArray(DATAROWS).forEach(dataRows::put); - cursor = response.optString(CURSOR); - } - - verifySchema(withoutCursorResponse.optJSONArray(SCHEMA), withCursorResponse.optJSONArray(SCHEMA)); - verifyDataRows(withoutCursorResponse.optJSONArray(DATAROWS), withCursorResponse.optJSONArray(DATAROWS)); - } - - public void verifySchema(JSONArray schemaOne, JSONArray schemaTwo) { - assertTrue(schemaOne.similar(schemaTwo)); - } - - public void verifyDataRows(JSONArray dataRowsOne, JSONArray dataRowsTwo) { - assertTrue(dataRowsOne.similar(dataRowsTwo)); - } - - private void enableCursorClusterSetting() throws IOException{ - updateClusterSettings(new ClusterSetting("persistent", "opendistro.sql.cursor.enabled", "true")); - } - - public String executeFetchAsStringQuery(String query, String fetchSize, String requestType) throws IOException { - String endpoint = "/_opendistro/_sql?format=" + requestType; - String requestBody = makeRequest(query, fetchSize); - - Request sqlRequest = new Request("POST", endpoint); - sqlRequest.setJsonEntity(requestBody); - - Response response = client().performRequest(sqlRequest); - String responseString = getResponseBody(response, true); - return responseString; - } - - private String makeRequest(String query, String fetch_size) { - return String.format("{" + - " \"fetch_size\": \"%s\"," + - " \"query\": \"%s\"" + - "}", fetch_size, query); - } - - private JSONObject executeJDBCRequest(String requestBody) throws IOException { - Request sqlRequest = getSqlRequest(requestBody, false, JDBC); - return new JSONObject(executeRequest(sqlRequest)); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/CustomExternalTestCluster.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/CustomExternalTestCluster.java deleted file mode 100644 index 981be35e76..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/CustomExternalTestCluster.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.network.NetworkModule; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.env.Environment; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.InternalTestCluster; -import org.elasticsearch.test.TestCluster; -import org.elasticsearch.transport.MockTransportClient; -import org.elasticsearch.transport.nio.MockNioTransportPlugin; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.elasticsearch.test.ESTestCase.getTestTransportType; - -public class CustomExternalTestCluster extends TestCluster { - - private static final Logger logger = LogManager.getLogger(CustomExternalTestCluster.class); - - private static final AtomicInteger counter = new AtomicInteger(); - public static final String EXTERNAL_CLUSTER_PREFIX = "external_"; - - private final MockTransportClient client; - - private final InetSocketAddress[] httpAddresses; - - private final String clusterName; - - private final int numDataNodes; - private final int numMasterAndDataNodes; - - public CustomExternalTestCluster(Path tempDir, Settings additionalSettings, Collection> pluginClasses, - TransportAddress... transportAddresses) { - super(0); - Settings.Builder clientSettingsBuilder = Settings.builder() - .put(additionalSettings) - .put("node.name", InternalTestCluster.TRANSPORT_CLIENT_PREFIX + EXTERNAL_CLUSTER_PREFIX + counter.getAndIncrement()) - .put("client.transport.ignore_cluster_name", true) - .put(Environment.PATH_HOME_SETTING.getKey(), tempDir); - boolean addMockTcpTransport = additionalSettings.get(NetworkModule.TRANSPORT_TYPE_KEY) == null; - - if (addMockTcpTransport) { - String transport = getTestTransportType(); - clientSettingsBuilder.put(NetworkModule.TRANSPORT_TYPE_KEY, transport); - if (pluginClasses.contains(MockNioTransportPlugin.class) == false) { - pluginClasses = new ArrayList<>(pluginClasses); - if (transport.equals(MockNioTransportPlugin.MOCK_NIO_TRANSPORT_NAME)) { - pluginClasses.add(MockNioTransportPlugin.class); - } - } - } - Settings clientSettings = clientSettingsBuilder.build(); - MockTransportClient client = new MockTransportClient(clientSettings, pluginClasses); - try { - client.addTransportAddresses(transportAddresses); - NodesInfoResponse nodeInfos = client.admin().cluster().prepareNodesInfo().clear().setSettings(true).setHttp(true).get(); - httpAddresses = new InetSocketAddress[nodeInfos.getNodes().size()]; - this.clusterName = nodeInfos.getClusterName().value(); - int dataNodes = 0; - int masterAndDataNodes = 0; - for (int i = 0; i < nodeInfos.getNodes().size(); i++) { - NodeInfo nodeInfo = nodeInfos.getNodes().get(i); - httpAddresses[i] = nodeInfo.getHttp().address().publishAddress().address(); - if (DiscoveryNode.isDataNode(nodeInfo.getSettings())) { - dataNodes++; - masterAndDataNodes++; - } else if (DiscoveryNode.isMasterNode(nodeInfo.getSettings())) { - masterAndDataNodes++; - } - } - this.numDataNodes = dataNodes; - this.numMasterAndDataNodes = masterAndDataNodes; - this.client = client; - - logger.info("Setup ExternalTestCluster [{}] made of [{}] nodes", nodeInfos.getClusterName().value(), size()); - } catch (Exception e) { - client.close(); - throw e; - } - } - - @Override - public void afterTest() { - - } - - @Override - public Client client() { - return client; - } - - @Override - public int size() { - return httpAddresses.length; - } - - @Override - public int numDataNodes() { - return numDataNodes; - } - - @Override - public int numDataAndMasterNodes() { - return numMasterAndDataNodes; - } - - @Override - public InetSocketAddress[] httpAddresses() { - return httpAddresses; - } - - @Override - public void close() throws IOException { - client.close(); - } - - /** - * This custom ExternalCluster class has ensureEstimatedStats() emptied out to prevent the transport client error - * made from making a request using a closed client after the tests are complete. - * */ - @Override - public void ensureEstimatedStats() {} - - @Override - public Iterable getClients() { - return Collections.singleton(client); - } - - @Override - public NamedWriteableRegistry getNamedWriteableRegistry() { - return client.getNamedWriteableRegistry(); - } - - @Override - public String getClusterName() { - return clusterName; - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/DateFormatIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/DateFormatIT.java deleted file mode 100644 index 85f8df4384..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/DateFormatIT.java +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.unittest.DateFormatTest; -import com.google.common.collect.Ordering; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; -import org.json.JSONArray; -import org.json.JSONObject; -import org.junit.Test; - -import java.io.IOException; -import java.util.List; -import java.util.Set; -import java.util.TreeSet; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.is; - -public class DateFormatIT extends SQLIntegTestCase { - - private static final String SELECT_FROM = - "SELECT insert_time " + - "FROM " + TestsConstants.TEST_INDEX_ONLINE + " "; - - @Override - protected void init() throws Exception { - loadIndex(Index.ONLINE); - } - - /** - * All of the following tests use UTC as their date_format timezone as this is the same timezone of the data - * being queried. This is to prevent discrepancies in the Elasticsearch query and the actual field data that is - * being checked for the integration tests. - * - * Large LIMIT values were given for some of these queries since the default result size of the query is 200 and - * this ends up excluding some of the expected values causing the assertion to fail. LIMIT overrides this. - */ - - @Test - public void equalTo() throws SqlParseException { - assertThat( - dateQuery(SELECT_FROM + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') = '2014-08-17'"), - contains("2014-08-17") - ); - } - - @Test - public void lessThan() throws SqlParseException { - assertThat( - dateQuery(SELECT_FROM + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') < '2014-08-18'"), - contains("2014-08-17") - ); - } - - @Test - public void lessThanOrEqualTo() throws SqlParseException { - assertThat( - dateQuery(SELECT_FROM + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') <= '2014-08-18' " + - "ORDER BY insert_time " + - "LIMIT 1000"), - contains("2014-08-17", "2014-08-18") - ); - } - - @Test - public void greaterThan() throws SqlParseException { - assertThat( - dateQuery(SELECT_FROM + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') > '2014-08-23'"), - contains("2014-08-24") - ); - } - - @Test - public void greaterThanOrEqualTo() throws SqlParseException { - assertThat( - dateQuery(SELECT_FROM + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') >= '2014-08-23' " + - "ORDER BY insert_time " + - "LIMIT 2000"), - contains("2014-08-23", "2014-08-24") - ); - } - - @Test - public void and() throws SqlParseException{ - assertThat( - dateQuery(SELECT_FROM + - "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') >= '2014-08-21' " + - "AND date_format(insert_time, 'yyyy-MM-dd', 'UTC') <= '2014-08-23' " + - "ORDER BY insert_time " + - "LIMIT 3000"), - contains("2014-08-21", "2014-08-22", "2014-08-23") - ); - } - - @Test - public void or() throws SqlParseException { - assertThat( - dateQuery(SELECT_FROM + - "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') < '2014-08-18' " + - "OR date_format(insert_time, 'yyyy-MM-dd', 'UTC') > '2014-08-23' " + - "ORDER BY insert_time " + - "LIMIT 1000"), - contains("2014-08-17", "2014-08-24") - ); - } - - - @Test - public void sortByDateFormat() throws IOException { - // Sort by expression in descending order, but sort inside in ascending order, so we increase our confidence - // that successful test isn't just random chance. - - JSONArray hits = - getHits(executeQuery("SELECT all_client, insert_time " + - " FROM " + TestsConstants.TEST_INDEX_ONLINE + - " ORDER BY date_format(insert_time, 'dd-MM-YYYY', 'UTC') DESC, insert_time " + - " LIMIT 10")); - - assertThat(new DateTime(getSource(hits.getJSONObject(0)).get("insert_time"), DateTimeZone.UTC), - is(new DateTime("2014-08-24T00:00:41.221Z", DateTimeZone.UTC))); - } - - @Test - public void sortByAliasedDateFormat() throws IOException { - JSONArray hits = - getHits(executeQuery("SELECT all_client, insert_time, date_format(insert_time, 'dd-MM-YYYY', 'UTC') date" + - " FROM " + TestsConstants.TEST_INDEX_ONLINE + - " ORDER BY date DESC, insert_time " + - " LIMIT 10")); - - assertThat(new DateTime(getSource(hits.getJSONObject(0)).get("insert_time"), DateTimeZone.UTC), - is(new DateTime("2014-08-24T00:00:41.221Z", DateTimeZone.UTC))); - } - - @Test - public void groupByAndSort() throws IOException { - JSONObject aggregations = executeQuery( - "SELECT date_format(insert_time, 'dd-MM-YYYY') " + - "FROM elasticsearch-sql_test_index_online " + - "GROUP BY date_format(insert_time, 'dd-MM-YYYY') " + - "ORDER BY date_format(insert_time, 'dd-MM-YYYY') DESC") - .getJSONObject("aggregations"); - - checkAggregations(aggregations, "date_format", Ordering.natural().reverse()); - } - - @Test - public void groupByAndSortAliasedReversed() throws IOException { - JSONObject aggregations = executeQuery( - "SELECT date_format(insert_time, 'dd-MM-YYYY') date " + - "FROM elasticsearch-sql_test_index_online " + - "GROUP BY date " + - "ORDER BY date DESC") - .getJSONObject("aggregations"); - - checkAggregations(aggregations, "date", Ordering.natural().reverse()); - } - - @Test - public void groupByAndSortAliased() throws IOException { - JSONObject aggregations = executeQuery( - "SELECT date_format(insert_time, 'dd-MM-YYYY') date " + - "FROM elasticsearch-sql_test_index_online " + - "GROUP BY date " + - "ORDER BY date ") - .getJSONObject("aggregations"); - - checkAggregations(aggregations, "date", Ordering.natural()); - } - - private void checkAggregations(JSONObject aggregations, String key, Ordering ordering) { - String date = DateFormatTest.getScriptAggregationKey(aggregations, key); - JSONArray buckets = aggregations.getJSONObject(date).getJSONArray("buckets"); - - assertThat(buckets.length(), is(8)); - - List aggregationSortKeys = IntStream.range(0, 8) - .mapToObj(index -> buckets.getJSONObject(index).getString("key")) - .collect(Collectors.toList()); - - assertTrue("The query result must be sorted by date in descending order", - ordering.isOrdered(aggregationSortKeys)); - } - - private Set dateQuery(String sql) throws SqlParseException { - try { - JSONObject response = executeQuery(sql); - return getResult(response, "insert_time"); - } catch (IOException e) { - throw new SqlParseException(String.format("Unable to process query '%s'", sql)); - } - } - - private Set getResult(JSONObject response, String fieldName) { - DateTimeFormatter formatter = DateTimeFormat.forPattern(TestsConstants.SIMPLE_DATE_FORMAT); - - JSONArray hits = getHits(response); - Set result = new TreeSet<>(); // Using TreeSet so order is maintained - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - JSONObject source = getSource(hit); - DateTime date = new DateTime(source.get(fieldName), DateTimeZone.UTC); - String formattedDate = formatter.print(date); - - result.add(formattedDate); - } - - return result; - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/DateFunctionsIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/DateFunctionsIT.java deleted file mode 100644 index 7d39f360b1..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/DateFunctionsIT.java +++ /dev/null @@ -1,324 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.search.SearchHit; -import org.joda.time.DateTime; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; -import org.json.JSONObject; -import org.junit.Test; - -import java.io.IOException; -import java.time.Month; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.lessThan; -import static org.hamcrest.Matchers.matchesPattern; - -public class DateFunctionsIT extends SQLIntegTestCase { - - private static final String FROM = "FROM " + TestsConstants.TEST_INDEX_ONLINE; - - /** - * Some of the first few SQL functions are tested in both SELECT and WHERE cases for flexibility and the remainder - * are merely tested in SELECT for simplicity. - * - * There is a limitation in all date SQL functions in that they expect a date field as input. In the future this - * can be expanded on by supporting CAST and casting dates given as Strings to TIMESTAMP (SQL's date type). - */ - - @Override - protected void init() throws Exception { - loadIndex(Index.ONLINE); - } - - @Test - public void year() throws IOException { - SearchHit[] hits = query( - "SELECT YEAR(insert_time) as year" - ); - for (SearchHit hit : hits) { - int year = (int) getField(hit, "year"); - DateTime insertTime = getDateFromSource(hit, "insert_time"); - assertThat(year, equalTo(insertTime.year().get())); - } - } - - @Test - public void monthOfYear() throws IOException { - SearchHit[] hits = query( - "SELECT MONTH_OF_YEAR(insert_time) as month_of_year" - ); - for (SearchHit hit : hits) { - int monthOfYear = (int) getField(hit, "month_of_year"); - DateTime insertTime = getDateFromSource(hit, "insert_time"); - assertThat(monthOfYear, equalTo(insertTime.monthOfYear().get())); - } - } - - @Test - public void weekOfYearInSelect() throws IOException { - SearchHit[] hits = query( - "SELECT WEEK_OF_YEAR(insert_time) as week_of_year" - ); - for (SearchHit hit : hits) { - int weekOfYear = (int) getField(hit, "week_of_year"); - DateTime insertTime = getDateFromSource(hit, "insert_time"); - assertThat(weekOfYear, equalTo(insertTime.weekOfWeekyear().get())); - } - } - - @Test - public void weekOfYearInWhere() throws IOException { - SearchHit[] hits = query( - "SELECT insert_time", - "WHERE DATE_FORMAT(insert_time, 'YYYY-MM-dd') < '2014-08-19' AND " + - "WEEK_OF_YEAR(insert_time) > 33", - "LIMIT 2000" - ); - for (SearchHit hit : hits) { - DateTime insertTime = getDateFromSource(hit, "insert_time"); - assertThat(insertTime.weekOfWeekyear().get(), greaterThan(33)); - } - } - - @Test - public void dayOfYearInSelect() throws IOException { - SearchHit[] hits = query( - "SELECT DAY_OF_YEAR(insert_time) as day_of_year", "LIMIT 2000" - ); - for (SearchHit hit : hits) { - int dayOfYear = (int) getField(hit, "day_of_year"); - DateTime insertTime = getDateFromSource(hit, "insert_time"); - assertThat(dayOfYear, equalTo(insertTime.dayOfYear().get())); - } - } - - @Test - public void dayOfYearInWhere() throws IOException { - SearchHit[] hits = query( - "SELECT insert_time", "WHERE DAY_OF_YEAR(insert_time) < 233", "LIMIT 10000" - ); - for (SearchHit hit : hits) { - DateTime insertTime = getDateFromSource(hit, "insert_time"); - assertThat(insertTime.dayOfYear().get(), lessThan(233)); - } - } - - @Test - public void dayOfMonthInSelect() throws IOException { - SearchHit[] hits = query( - "SELECT DAY_OF_MONTH(insert_time) as day_of_month", "LIMIT 2000" - ); - for (SearchHit hit : hits) { - int dayOfMonth = (int) getField(hit, "day_of_month"); - DateTime insertTime = getDateFromSource(hit, "insert_time"); - assertThat(dayOfMonth, equalTo(insertTime.dayOfMonth().get())); - } - } - - @Test - public void dayOfMonthInWhere() throws IOException { - SearchHit[] hits = query( - "SELECT insert_time", "WHERE DAY_OF_MONTH(insert_time) < 21", "LIMIT 10000" - ); - for (SearchHit hit : hits) { - DateTime insertTime = getDateFromSource(hit, "insert_time"); - assertThat(insertTime.dayOfMonth().get(), lessThan(21)); - } - } - - @Test - public void dayOfWeek() throws IOException { - SearchHit[] hits = query( - "SELECT DAY_OF_WEEK(insert_time) as day_of_week", "LIMIT 2000" - ); - for (SearchHit hit : hits) { - int dayOfWeek = (int) getField(hit, "day_of_week"); - DateTime insertTime = getDateFromSource(hit, "insert_time"); - assertThat(dayOfWeek, equalTo(insertTime.dayOfWeek().get())); - } - } - - @Test - public void hourOfDay() throws IOException { - SearchHit[] hits = query( - "SELECT HOUR_OF_DAY(insert_time) as hour_of_day", "LIMIT 1000" - ); - for (SearchHit hit : hits) { - int hourOfDay = (int) getField(hit, "hour_of_day"); - DateTime insertTime = getDateFromSource(hit, "insert_time"); - assertThat(hourOfDay, equalTo(insertTime.hourOfDay().get())); - } - } - - @Test - public void minuteOfDay() throws IOException { - SearchHit[] hits = query( - "SELECT MINUTE_OF_DAY(insert_time) as minute_of_day", "LIMIT 500" - ); - for (SearchHit hit : hits) { - int minuteOfDay = (int) getField(hit, "minute_of_day"); - DateTime insertTime = getDateFromSource(hit, "insert_time"); - assertThat(minuteOfDay, equalTo(insertTime.minuteOfDay().get())); - } - } - - @Test - public void minuteOfHour() throws IOException { - SearchHit[] hits = query( - "SELECT MINUTE_OF_HOUR(insert_time) as minute_of_hour", "LIMIT 500" - ); - for (SearchHit hit : hits) { - int minuteOfHour = (int) getField(hit, "minute_of_hour"); - DateTime insertTime = getDateFromSource(hit, "insert_time"); - assertThat(minuteOfHour, equalTo(insertTime.minuteOfHour().get())); - } - } - - @Test - public void secondOfMinute() throws IOException { - SearchHit[] hits = query( - "SELECT SECOND_OF_MINUTE(insert_time) as second_of_minute", "LIMIT 500" - ); - for (SearchHit hit : hits) { - int secondOfMinute = (int) getField(hit, "second_of_minute"); - DateTime insertTime = getDateFromSource(hit, "insert_time"); - assertThat(secondOfMinute, equalTo(insertTime.secondOfMinute().get())); - } - } - - @Test - public void month() throws IOException { - SearchHit[] hits = query( - "SELECT MONTH(insert_time) AS month", "LIMIT 500" - ); - for (SearchHit hit: hits) { - int month = (int) getField(hit, "month"); - DateTime dateTime = getDateFromSource(hit, "insert_time"); - assertThat(month, equalTo(dateTime.monthOfYear().get())); - } - } - - @Test - public void dayofmonth() throws IOException { - SearchHit[] hits = query( - "SELECT DAYOFMONTH(insert_time) AS dayofmonth", "LIMIT 500" - ); - for (SearchHit hit: hits) { - int dayofmonth = (int) getField(hit, "dayofmonth"); - DateTime dateTime = getDateFromSource(hit, "insert_time"); - assertThat(dayofmonth, equalTo(dateTime.dayOfMonth().get())); - } - } - - @Test - public void date() throws IOException { - SearchHit[] hits = query( - "SELECT DATE(insert_time) AS date", "LIMIT 500" - ); - for (SearchHit hit: hits) { - String date = (String) getField(hit, "date"); - DateTime dateTime = getDateFromSource(hit, "insert_time"); - assertThat(date, equalTo(dateTime.toString("yyyy-MM-dd"))); - } - } - - @Test - public void monthname() throws IOException { - SearchHit[] hits = query( - "SELECT MONTHNAME(insert_time) AS monthname", "LIMIT 500" - ); - for (SearchHit hit: hits) { - String monthname = (String) getField(hit, "monthname"); - DateTime dateTime = getDateFromSource(hit, "insert_time"); - assertThat(Month.valueOf(monthname), equalTo(Month.of(dateTime.getMonthOfYear()))); - } - } - - @Test - public void timestamp() throws IOException { - SearchHit[] hits = query( - "SELECT TIMESTAMP(insert_time) AS timestamp", "LIMIT 500" - ); - for (SearchHit hit: hits) { - String timastamp = (String) getField(hit, "timestamp"); - DateTime dateTime = getDateFromSource(hit, "insert_time"); - assertThat(timastamp, equalTo(dateTime.toString("yyyy-MM-dd HH:mm:ss"))); - } - } - - @Test - public void maketime() throws IOException { - SearchHit[] hits = query("SELECT MAKETIME(13, 1, 1) AS maketime"); - String maketime = (String) getField(hits[0], "maketime"); - assertThat(maketime, equalTo("13:01:01")); - } - - @Test - public void now() throws IOException { - SearchHit[] hits = query("SELECT NOW() AS now"); - String now = (String) getField(hits[0], "now"); - assertThat(now, matchesPattern("[0-9]{2}:[0-9]{2}:[0-9]{2}")); - } - - @Test - public void curdate() throws IOException { - SearchHit[] hits = query("SELECT CURDATE() AS curdate"); - String curdate = (String) getField(hits[0], "curdate"); - assertThat(curdate, matchesPattern("[0-9]{4}-[0-9]{2}-[0-9]{2}")); - } - - private SearchHit[] query(String select, String... statements) throws IOException { - return execute(select + " " + FROM + " " + String.join(" ", statements)); - } - - // TODO: I think this code is now re-used in multiple classes, would be good to move to the base class. - private SearchHit[] execute(String sqlRequest) throws IOException { - final JSONObject jsonObject = executeRequest(makeRequest(sqlRequest)); - - final XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser( - NamedXContentRegistry.EMPTY, - LoggingDeprecationHandler.INSTANCE, - jsonObject.toString()); - return SearchResponse.fromXContent(parser).getHits().getHits(); - } - - private Object getField(SearchHit hit, String fieldName) { - return hit.field(fieldName).getValue(); - } - - private Object getFieldFromSource(SearchHit hit, String fieldName) { - return hit.getSourceAsMap().get(fieldName); - } - - private DateTime getDateTime(String date) { - DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); - return formatter.parseDateTime(date); - } - - private DateTime getDateFromSource(SearchHit hit, String dateField) { - return getDateTime((String) getFieldFromSource(hit, dateField)); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/DeleteIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/DeleteIT.java deleted file mode 100644 index ee3dd766e4..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/DeleteIT.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; -import org.json.JSONObject; -import org.junit.Test; - -import java.io.IOException; - -import static org.hamcrest.core.IsEqual.equalTo; - -public class DeleteIT extends SQLIntegTestCase { - - @Override - protected void init() throws Exception { - loadIndex(Index.ACCOUNT); - loadIndex(Index.PHRASE); - } - - @Test - public void deleteAllTest() throws IOException, InterruptedException { - String selectQuery = StringUtils.format("SELECT * FROM %s", TestsConstants.TEST_INDEX_ACCOUNT); - JSONObject response = executeRequest(makeRequest(selectQuery)); - int totalHits = getTotalHits(response); - - String deleteQuery = StringUtils.format("DELETE FROM %s", TestsConstants.TEST_INDEX_ACCOUNT); - response = executeRequest(makeRequest(deleteQuery)); - assertThat(response.getInt("deleted"), equalTo(totalHits)); - - // The documents are not deleted immediately, causing the next search call to return all results. - // To prevent flakiness, the minimum value of 2000 msec works fine. - Thread.sleep(2000); - - response = executeRequest(makeRequest(selectQuery)); - assertThat(getTotalHits(response), equalTo(0)); - } - - @Test - public void deleteWithConditionTest() throws IOException, InterruptedException { - String selectQuery = StringUtils.format( - "SELECT * FROM %s WHERE match_phrase(phrase, 'quick fox here')", - TestsConstants.TEST_INDEX_PHRASE - ); - JSONObject response = executeRequest(makeRequest(selectQuery)); - int totalHits = getTotalHits(response); - - String deleteQuery = StringUtils.format( - "DELETE FROM %s WHERE match_phrase(phrase, 'quick fox here')", - TestsConstants.TEST_INDEX_PHRASE - ); - response = executeRequest(makeRequest(deleteQuery)); - assertThat(response.getInt("deleted"), equalTo(totalHits)); - // The documents are not deleted immediately, causing the next search call to return all results. - // To prevent flakiness, the minimum value of 2000 msec works fine. - Thread.sleep(2000); - - selectQuery = StringUtils.format("SELECT * FROM %s", TestsConstants.TEST_INDEX_PHRASE); - - response = executeRequest(makeRequest(selectQuery)); - assertThat(getTotalHits(response), equalTo(5)); - } - - @Test - public void deleteAllWithJdbcFormat() throws IOException, InterruptedException { - String selectQuery = StringUtils.format("SELECT * FROM %s", TestsConstants.TEST_INDEX_ACCOUNT); - JSONObject response = executeRequest(makeRequest(selectQuery)); - int totalHits = getTotalHits(response); - - String deleteQuery = StringUtils.format("DELETE FROM %s", TestsConstants.TEST_INDEX_ACCOUNT); - - response = new JSONObject(executeQuery(deleteQuery, "jdbc")); - System.out.println(response); - assertThat(response.query("/schema/0/name"), equalTo("deleted_rows")); - assertThat(response.query("/schema/0/type"), equalTo("long")); - assertThat(response.query("/datarows/0/0"), equalTo(totalHits)); - assertThat(response.query("/total"), equalTo(1)); - assertThat(response.query("/status"), equalTo(200)); - assertThat(response.query("/size"), equalTo(1)); - - // The documents are not deleted immediately, causing the next search call to return all results. - // To prevent flakiness, the minimum value of 2000 msec works fine. - Thread.sleep(2000); - - response = executeRequest(makeRequest(selectQuery)); - assertThat(getTotalHits(response), equalTo(0)); - - // Multiple invocation of delete query should return deleted == 0 - response = new JSONObject(executeQuery(deleteQuery, "jdbc")); - assertThat(response.query("/datarows/0/0"), equalTo(0)); - } - - @Test - public void deleteWithConditionTestJdbcFormat() throws IOException, InterruptedException { - String selectQuery = StringUtils.format( - "SELECT * FROM %s WHERE match_phrase(phrase, 'quick fox here')", - TestsConstants.TEST_INDEX_PHRASE - ); - - JSONObject response = executeRequest(makeRequest(selectQuery)); - int totalHits = getTotalHits(response); - - String deleteQuery = StringUtils.format( - "DELETE FROM %s WHERE match_phrase(phrase, 'quick fox here')", - TestsConstants.TEST_INDEX_PHRASE - ); - - response = new JSONObject(executeQuery(deleteQuery, "jdbc")); - System.out.println(response); - assertThat(response.query("/schema/0/name"), equalTo("deleted_rows")); - assertThat(response.query("/schema/0/type"), equalTo("long")); - assertThat(response.query("/datarows/0/0"), equalTo(totalHits)); - assertThat(response.query("/total"), equalTo(1)); - assertThat(response.query("/status"), equalTo(200)); - assertThat(response.query("/size"), equalTo(1)); - - // The documents are not deleted immediately, causing the next search call to return all results. - // To prevent flakiness, the minimum value of 2000 msec works fine. - Thread.sleep(2000); - - selectQuery = StringUtils.format("SELECT * FROM %s", TestsConstants.TEST_INDEX_PHRASE); - - response = executeRequest(makeRequest(selectQuery)); - assertThat(getTotalHits(response), equalTo(5)); - - // Multiple invocation of delete query should return deleted == 0 - response = new JSONObject(executeQuery(deleteQuery, "jdbc")); - assertThat(response.query("/datarows/0/0"), equalTo(0)); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ExplainIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ExplainIT.java deleted file mode 100644 index a2ff2ab684..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ExplainIT.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import com.google.common.io.Files; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; - -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_ACCOUNT; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_DOG; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_LOCATION; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_NESTED_TYPE; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_PEOPLE; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_PHRASE; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; - -public class ExplainIT extends SQLIntegTestCase { - - @Override - protected void init() throws Exception { - loadIndex(Index.ACCOUNT); - loadIndex(Index.DOG); - loadIndex(Index.PEOPLE); - loadIndex(Index.PHRASE); - loadIndex(Index.LOCATION); - loadIndex(Index.NESTED); - } - - @Test - public void searchSanity() throws IOException { - - String expectedOutputFilePath = TestUtils.getResourceFilePath( - "src/test/resources/expectedOutput/search_explain.json"); - String expectedOutput = Files.toString(new File(expectedOutputFilePath), StandardCharsets.UTF_8) - .replaceAll("\r",""); - - String result = explainQuery(String.format("SELECT * FROM %s WHERE firstname LIKE 'A%%' AND age > 20 " + - "GROUP BY gender order by _score", TEST_INDEX_ACCOUNT)); - Assert.assertThat(result.replaceAll("\\s+",""), equalTo(expectedOutput.replaceAll("\\s+",""))); - } - - // This test was ignored because group by case function is not supported - @Ignore - @Test - public void aggregationQuery() throws IOException { - - String expectedOutputFilePath = TestUtils.getResourceFilePath( - "src/test/resources/expectedOutput/aggregation_query_explain.json"); - String expectedOutput = Files.toString(new File(expectedOutputFilePath), StandardCharsets.UTF_8) - .replaceAll("\r",""); - - String result = explainQuery(String.format("SELECT address, CASE WHEN gender='0' then 'aaa' else 'bbb'end a2345," + - "count(age) FROM %s GROUP BY terms('field'='address','execution_hint'='global_ordinals'),a2345", - TEST_INDEX_ACCOUNT)); - Assert.assertThat(result.replaceAll("\\s+",""), equalTo(expectedOutput.replaceAll("\\s+",""))); - } - - @Test - public void explainScriptValue() throws IOException { - - String expectedOutputFilePath = TestUtils.getResourceFilePath( - "src/test/resources/expectedOutput/script_value.json"); - String expectedOutput = Files.toString(new File(expectedOutputFilePath), StandardCharsets.UTF_8) - .replaceAll("\r",""); - - String result = explainQuery(String.format("SELECT case when gender is null then 'aaa' " + - "else gender end test , account_number FROM %s", TEST_INDEX_ACCOUNT)); - Assert.assertThat(result.replaceAll("\\s+",""), equalTo(expectedOutput.replaceAll("\\s+",""))); - } - - @Test - public void betweenScriptValue() throws IOException { - - String expectedOutputFilePath = TestUtils.getResourceFilePath( - "src/test/resources/expectedOutput/between_query.json"); - String expectedOutput = Files.toString(new File(expectedOutputFilePath), StandardCharsets.UTF_8) - .replaceAll("\r",""); - - String result = explainQuery(String.format("SELECT case when balance between 100 and 200 then 'aaa' " + - "else balance end test, account_number FROM %s", TEST_INDEX_ACCOUNT)); - Assert.assertThat(result.replaceAll("\\s+",""), equalTo(expectedOutput.replaceAll("\\s+",""))); - } - - @Test - public void searchSanityFilter() throws IOException { - - String expectedOutputFilePath = TestUtils.getResourceFilePath( - "src/test/resources/expectedOutput/search_explain_filter.json"); - String expectedOutput = Files.toString(new File(expectedOutputFilePath), StandardCharsets.UTF_8) - .replaceAll("\r",""); - - String result = explainQuery(String.format("SELECT * FROM %s WHERE firstname LIKE 'A%%' " + - "AND age > 20 GROUP BY gender", TEST_INDEX_ACCOUNT)); - Assert.assertThat(result.replaceAll("\\s+",""), equalTo(expectedOutput.replaceAll("\\s+",""))); - } - - @Test - public void deleteSanity() throws IOException { - - String expectedOutputFilePath = TestUtils.getResourceFilePath( - "src/test/resources/expectedOutput/delete_explain.json"); - String expectedOutput = Files.toString(new File(expectedOutputFilePath), StandardCharsets.UTF_8) - .replaceAll("\r","");; - - String result = explainQuery(String.format("DELETE FROM %s WHERE firstname LIKE 'A%%' AND age > 20", - TEST_INDEX_ACCOUNT)); - Assert.assertThat(result.replaceAll("\\s+",""), equalTo(expectedOutput.replaceAll("\\s+",""))); - } - - @Test - public void spatialFilterExplainTest() throws IOException { - - String expectedOutputFilePath = TestUtils.getResourceFilePath( - "src/test/resources/expectedOutput/search_spatial_explain.json"); - String expectedOutput = Files.toString(new File(expectedOutputFilePath), StandardCharsets.UTF_8) - .replaceAll("\r","");; - - String result = explainQuery(String.format("SELECT * FROM %s WHERE GEO_INTERSECTS" + - "(place,'POLYGON ((102 2, 103 2, 103 3, 102 3, 102 2))')", TEST_INDEX_LOCATION)); - Assert.assertThat(result.replaceAll("\\s+",""), equalTo(expectedOutput.replaceAll("\\s+",""))); - } - - @Test - public void orderByOnNestedFieldTest() throws Exception { - - String result = explainQuery(String.format("SELECT * FROM %s ORDER BY NESTED('message.info','message')", - TEST_INDEX_NESTED_TYPE)); - Assert.assertThat(result.replaceAll("\\s+", ""), - equalTo("{\"from\":0,\"size\":200,\"sort\":[{\"message.info\":" + - "{\"order\":\"asc\",\"nested\":{\"path\":\"message\"}}}]}")); - } - - @Test - public void multiMatchQuery() throws IOException { - - String expectedOutputFilePath = TestUtils.getResourceFilePath( - "src/test/resources/expectedOutput/multi_match_query.json"); - String expectedOutput = Files.toString(new File(expectedOutputFilePath), StandardCharsets.UTF_8) - .replaceAll("\r", ""); - - String result = explainQuery(String.format("SELECT * FROM %s WHERE multimatch('query'='this is a test'," + - "'fields'='subject^3,message','analyzer'='standard','type'='best_fields','boost'=1.0," + - "'slop'=0,'tie_breaker'=0.3,'operator'='and')", TEST_INDEX_ACCOUNT)); - Assert.assertThat(result.replaceAll("\\s+", ""), equalTo(expectedOutput.replaceAll("\\s+", ""))); - } - - @Test - public void termsIncludeExcludeExplainTest() throws IOException { - - final String queryPrefix = "SELECT * FROM " + TEST_INDEX_PHRASE + " GROUP BY "; - final String expected1 = "\"include\":\".*sport.*\",\"exclude\":\"water_.*\""; - final String expected2 = "\"include\":[\"honda\",\"mazda\"],\"exclude\":[\"jensen\",\"rover\"]"; - final String expected3 = "\"include\":{\"partition\":0,\"num_partitions\":20}"; - - String result = explainQuery(queryPrefix + " terms('field'='correspond_brand_name','size'='10'," + - "'alias'='correspond_brand_name','include'='\\\".*sport.*\\\"','exclude'='\\\"water_.*\\\"')"); - Assert.assertThat(result, containsString(expected1)); - - result = explainQuery(queryPrefix + "terms('field'='correspond_brand_name','size'='10'," + - "'alias'='correspond_brand_name','include'='[\\\"mazda\\\", \\\"honda\\\"]'," + - "'exclude'='[\\\"rover\\\", \\\"jensen\\\"]')"); - Assert.assertThat(result, containsString(expected2)); - - result = explainQuery(queryPrefix + "terms('field'='correspond_brand_name','size'='10'," + - "'alias'='correspond_brand_name','include'='{\\\"partition\\\":0,\\\"num_partitions\\\":20}')"); - Assert.assertThat(result, containsString(expected3)); - } - - @Test - public void explainNLJoin() throws IOException { - - String expectedOutputFilePath = TestUtils.getResourceFilePath( - "src/test/resources/expectedOutput/nested_loop_join_explain.json"); - String expectedOutput = Files.toString(new File(expectedOutputFilePath), StandardCharsets.UTF_8) - .replaceAll("\r", ""); - - String query = "SELECT /*! USE_NL*/ a.firstname ,a.lastname , a.gender ,d.dog_name FROM " + - TEST_INDEX_PEOPLE + "/people a JOIN " + TEST_INDEX_DOG + "/dog d on d.holdersName = a.firstname" + - " WHERE (a.age > 10 OR a.balance > 2000) AND d.age > 1"; - String result = explainQuery(query); - - Assert.assertThat(result.replaceAll("\\s+", ""), equalTo(expectedOutput.replaceAll("\\s+", ""))); - } - - public void testContentTypeOfExplainRequestShouldBeJson() throws IOException { - String query = makeRequest("SELECT firstname FROM elasticsearch-sql_test_index_account"); - Request request = getSqlRequest(query, true); - - Response response = client().performRequest(request); - - assertEquals("application/json; charset=UTF-8", response.getHeader("content-type")); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/GetEndpointQueryIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/GetEndpointQueryIT.java deleted file mode 100644 index cb7b67f8cd..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/GetEndpointQueryIT.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import org.elasticsearch.client.ResponseException; -import org.junit.Rule; -import org.junit.Test; - -import java.io.IOException; -import org.junit.rules.ExpectedException; - -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_ACCOUNT; - -/** - * Tests to cover requests with "?format=csv" parameter - */ -public class GetEndpointQueryIT extends SQLIntegTestCase { - - @Rule - public ExpectedException rule = ExpectedException.none(); - - @Override - protected void init() throws Exception { - loadIndex(Index.ACCOUNT); - } - - @Test - public void getEndPointShouldBeInvalid() throws IOException { - rule.expect(ResponseException.class); - rule.expectMessage("Incorrect HTTP method"); - String query = "select name from " + TEST_INDEX_ACCOUNT; - executeQueryWithGetRequest(query); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/HashJoinIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/HashJoinIT.java deleted file mode 100644 index f883505208..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/HashJoinIT.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import org.json.JSONArray; -import org.json.JSONObject; -import org.junit.Assert; -import org.junit.Test; - -import java.io.IOException; -import java.util.HashSet; -import java.util.Locale; -import java.util.Set; - -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_ACCOUNT; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_GAME_OF_THRONES; -import static org.hamcrest.Matchers.equalTo; - -/** - * Test new hash join algorithm by comparison with old implementation. - */ -public class HashJoinIT extends SQLIntegTestCase { - - /** - * Hint to use old join algorithm - */ - private static final String USE_OLD_JOIN_ALGORITHM = "/*! USE_NL*/"; - - /** - * Set limit to 100% to bypass circuit break check - */ - private static final String BYPASS_CIRCUIT_BREAK = "/*! JOIN_CIRCUIT_BREAK_LIMIT(100)*/"; - - /** - * Enable term filter optimization - */ - private static final String ENABLE_TERMS_FILTER = "/*! HASH_WITH_TERMS_FILTER*/"; - - /** - * Default page size > block size - */ - private static final String PAGE_SIZE_GREATER_THAN_BLOCK_SIZE = "/*! JOIN_ALGORITHM_BLOCK_SIZE(5)*/"; - - /** - * Page size < block size - */ - private static final String PAGE_SIZE_LESS_THAN_BLOCK_SIZE = - "/*! JOIN_ALGORITHM_BLOCK_SIZE(5)*/ /*! JOIN_SCROLL_PAGE_SIZE(2)*/"; - - @Override - protected void init() throws Exception { - loadIndex(Index.ACCOUNT); - loadIndex(Index.GAME_OF_THRONES); - } - - @Test - public void innerJoin() throws IOException { - - testJoin("INNER JOIN"); - } - - @Test - public void leftJoin() throws IOException { - - testJoin("LEFT JOIN"); - } - - @Test - public void innerJoinWithObjectField() throws IOException { - testJoinWithObjectField("INNER JOIN", ""); - } - - @Test - public void leftJoinWithObjectField() throws IOException { - testJoinWithObjectField("LEFT JOIN", ""); - } - - @Test - public void innerJoinWithObjectFieldAndTermsFilter() throws IOException { - testJoinWithObjectField("INNER JOIN", ENABLE_TERMS_FILTER); - } - - @Test - public void leftJoinWithObjectFieldAndTermsFilter() throws IOException { - testJoinWithObjectField("LEFT JOIN", ENABLE_TERMS_FILTER); - } - - @Test - public void innerJoinWithObjectFieldAndPageSizeGreaterThanBlockSize() throws IOException { - testJoinWithObjectField("INNER JOIN", PAGE_SIZE_GREATER_THAN_BLOCK_SIZE); - } - - @Test - public void leftJoinWithObjectFieldAndPageSizeGreaterThanBlockSize() throws IOException { - testJoinWithObjectField("LEFT JOIN", PAGE_SIZE_GREATER_THAN_BLOCK_SIZE); - } - - @Test - public void innerJoinWithObjectFieldAndPageSizeLessThanBlockSize() throws IOException { - testJoinWithObjectField("INNER JOIN", PAGE_SIZE_LESS_THAN_BLOCK_SIZE); - } - - @Test - public void leftJoinWithObjectFieldAndPageSizeLessThanBlockSize() throws IOException { - testJoinWithObjectField("LEFT JOIN", PAGE_SIZE_LESS_THAN_BLOCK_SIZE); - } - - private void testJoin(final String join) throws IOException { - - final String queryPrefix = "SELECT"; - - // TODO: reduce the balance threshold to 10000 when the memory circuit breaker issue - // (https://github.com/opendistro-for-elasticsearch/sql/issues/73) is fixed. - final String querySuffixTemplate = "a.firstname, a.lastname, b.city, b.state FROM %1$s a %2$s %1$s b " + - "ON b.age = a.age WHERE a.balance > 45000 AND b.age > 25 LIMIT 1000000"; - final String querySuffix = String.format(Locale.ROOT, querySuffixTemplate, TEST_INDEX_ACCOUNT, join); - - final String oldQuery = String.join(" ", queryPrefix, USE_OLD_JOIN_ALGORITHM, querySuffix); - final String newQuery = String.join(" ", queryPrefix, BYPASS_CIRCUIT_BREAK, querySuffix); - - executeAndCompareOldAndNewJoins(oldQuery, newQuery); - } - - private void testJoinWithObjectField(final String join, final String hint) throws IOException { - - final String queryPrefix = "SELECT"; - - // TODO: reduce the balance threshold to 10000 when the memory circuit breaker issue - // (https://github.com/opendistro-for-elasticsearch/sql/issues/73) is fixed. - final String querySuffixTemplate = "c.name.firstname, c.name.lastname, f.hname, f.seat " + - "FROM %1$s c %2$s %1$s f ON f.gender.keyword = c.gender.keyword " + - "AND f.house.keyword = c.house.keyword " + - "WHERE c.gender = 'M' LIMIT 1000000"; - final String querySuffix = String.format(Locale.ROOT, querySuffixTemplate, TEST_INDEX_GAME_OF_THRONES, join); - - final String oldQuery = String.join(" ", queryPrefix, USE_OLD_JOIN_ALGORITHM, querySuffix); - final String newQuery = String.join(" ", queryPrefix, hint, BYPASS_CIRCUIT_BREAK, querySuffix); - - executeAndCompareOldAndNewJoins(oldQuery, newQuery); - } - - private void executeAndCompareOldAndNewJoins(final String oldQuery, final String newQuery) throws IOException { - - final JSONObject responseOld = executeQuery(oldQuery); - final JSONObject responseNew = executeQuery(newQuery); - - Assert.assertThat(getTotalHits(responseOld), equalTo(getTotalHits(responseNew))); - - final JSONArray hitsOld = getHits(responseOld); - final JSONArray hitsNew = getHits(responseNew); - - Assert.assertThat(hitsOld.length(), equalTo(hitsNew.length())); - - Set idsOld = new HashSet<>(); - - hitsOld.forEach(hitObj -> { - JSONObject hit = (JSONObject) hitObj; - idsOld.add(hit.getString("_id")); - }); - - hitsNew.forEach(hitObj -> { - JSONObject hit = (JSONObject) hitObj; - Assert.assertTrue(idsOld.contains(hit.getString("_id"))); - }); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/HavingIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/HavingIT.java deleted file mode 100644 index 83f6fb8697..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/HavingIT.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import org.hamcrest.Matcher; -import org.json.JSONArray; -import org.json.JSONObject; -import org.junit.Test; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.hamcrest.Matchers.arrayContaining; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.is; - -public class HavingIT extends SQLIntegTestCase { - - private static final String SELECT_FROM_WHERE_GROUP_BY = - "SELECT state, COUNT(*) cnt " + - "FROM " + TestsConstants.TEST_INDEX_ACCOUNT + " " + - "WHERE age = 30 " + - "GROUP BY state "; - - private static final Set> states1 = rowSet(1, Arrays.asList( - "AK", "AR", "CT", "DE", "HI", "IA", "IL", "IN", "LA", "MA", "MD", "MN", - "MO", "MT", "NC", "ND", "NE", "NH", "NJ", "NV", "SD", "VT", "WV", "WY" - )); - private static final Set> states2 = rowSet(2, Arrays.asList("AZ", "DC", "KS", "ME")); - private static final Set> states3 = rowSet(3, Arrays.asList("AL", "ID", "KY", "OR", "TN")); - - @Override - protected void init() throws Exception { - loadIndex(Index.ACCOUNT); - } - - @Test - public void equalsTo() throws IOException { - assertThat( - query(SELECT_FROM_WHERE_GROUP_BY + "HAVING cnt = 2"), - resultSet( - states2 - ) - ); - } - - @Test - public void lessThanOrEqual() throws IOException { - assertThat( - query(SELECT_FROM_WHERE_GROUP_BY + "HAVING cnt <= 2"), - resultSet( - states1, - states2 - ) - ); - } - - @Test - public void notEqualsTo() throws IOException { - assertThat( - query(SELECT_FROM_WHERE_GROUP_BY + "HAVING cnt <> 2"), - resultSet( - states1, - states3 - ) - ); - } - - @Test - public void between() throws IOException { - assertThat( - query(SELECT_FROM_WHERE_GROUP_BY + "HAVING cnt BETWEEN 1 AND 2"), - resultSet( - states1, - states2 - ) - ); - } - - @Test - public void notBetween() throws IOException { - assertThat( - query(SELECT_FROM_WHERE_GROUP_BY + "HAVING cnt NOT BETWEEN 1 AND 2"), - resultSet( - states3 - ) - ); - } - - @Test - public void in() throws IOException { - assertThat( - query(SELECT_FROM_WHERE_GROUP_BY + "HAVING cnt IN (2, 3)"), - resultSet( - states2, - states3 - ) - ); - } - - @Test - public void notIn() throws IOException { - assertThat( - query(SELECT_FROM_WHERE_GROUP_BY + "HAVING cnt NOT IN (2, 3)"), - resultSet( - states1 - ) - ); - } - - @Test - public void and() throws IOException { - assertThat( - query(SELECT_FROM_WHERE_GROUP_BY + "HAVING cnt >= 1 AND cnt < 3"), - resultSet( - states1, - states2 - ) - ); - } - - @Test - public void or() throws IOException { - assertThat( - query(SELECT_FROM_WHERE_GROUP_BY + "HAVING cnt = 1 OR cnt = 3"), - resultSet( - states1, - states3 - ) - ); - } - - @Test - public void not() throws IOException { - assertThat( - query(SELECT_FROM_WHERE_GROUP_BY + "HAVING NOT cnt >= 2"), - resultSet( - states1 - ) - ); - } - - @Test - public void notAndOr() throws IOException { - assertThat( - query(SELECT_FROM_WHERE_GROUP_BY + "HAVING NOT (cnt > 0 AND cnt <= 2)"), - resultSet( - states3 - ) - ); - } - - private Set query(String query) throws IOException { - JSONObject response = executeQuery(query); - return getResult(response, "state.keyword", "cnt"); - } - - private Set getResult(JSONObject response, String aggName, String aggFunc) { - - String bucketsPath = String.format(Locale.ROOT, "/aggregations/%s/buckets", aggName); - JSONArray buckets = (JSONArray) response.query(bucketsPath); - - Set result = new HashSet<>(); - for (int i = 0; i < buckets.length(); i++) { - JSONObject bucket = buckets.getJSONObject(i); - result.add(new Object[]{ - bucket.get("key"), - ((JSONObject) bucket.get(aggFunc)).getLong("value") - }); - } - - return result; - } - - @SafeVarargs - private final Matcher> resultSet(Set>... rowSets) { - return containsInAnyOrder(Arrays.stream(rowSets) - .flatMap(Collection::stream) - .collect(Collectors.toList())); - } - - private static Set> rowSet(long count, List states) { - return states.stream() - .map(state -> row(state, count)) - .collect(Collectors.toSet()); - } - - private static Matcher row(String state, long count) { - return arrayContaining(is(state), is(count)); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/JSONRequestIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/JSONRequestIT.java deleted file mode 100644 index 8bdf2ff9b9..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/JSONRequestIT.java +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHits; -import org.json.JSONObject; -import org.junit.Test; - -import java.io.IOException; -import java.util.Map; - -import static org.hamcrest.CoreMatchers.anyOf; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.lessThan; - -public class JSONRequestIT extends SQLIntegTestCase { - - @Override - protected void init() throws Exception { - loadIndex(Index.ACCOUNT); - loadIndex(Index.NESTED); - } - - @Test - public void search() throws IOException { - int ageToCompare = 25; - SearchHits response = query(String.format("{\"query\":\"" + - "SELECT * " + - "FROM %s " + - "WHERE age > %s " + - "LIMIT 1000\"}", TestsConstants.TEST_INDEX_ACCOUNT, ageToCompare)); - SearchHit[] hits = response.getHits(); - for (SearchHit hit : hits) { - int age = (int) hit.getSourceAsMap().get("age"); - assertThat(age, greaterThan(ageToCompare)); - } - } - - @Test - public void searchWithFilterAndNoWhere() throws IOException { - /* - * Human readable format of the request defined below: - * { - * "query": "SELECT * FROM accounts LIMIT 1000", - * "filter": { - * "range": { - * "age": { - * "gt": 25 - * } - * } - * } - * } - */ - int ageToCompare = 25; - SearchHits response = query(String.format("{\"query\":\"" + - "SELECT * " + - "FROM %s " + - "LIMIT 1000\",\"filter\":{\"range\":{\"age\":{\"gt\":%s}}}}", TestsConstants.TEST_INDEX_ACCOUNT, ageToCompare)); - SearchHit[] hits = response.getHits(); - for (SearchHit hit : hits) { - int age = (int) hit.getSourceAsMap().get("age"); - assertThat(age, greaterThan(ageToCompare)); - } - } - - @Test - public void searchWithRangeFilter() throws IOException { - /* - * Human readable format of the request defined below: - * { - * "query": "SELECT * FROM accounts WHERE age > 25 LIMIT 1000", - * "filter": { - * "range": { - * "balance": { - * "lt": 35000 - * } - * } - * } - * } - */ - int ageToCompare = 25; - int balanceToCompare = 35000; - SearchHits response = query(String.format("{\"query\":\"" + - "SELECT * " + - "FROM %s " + - "WHERE age > %s " + - "LIMIT 1000\",\"filter\":{\"range\":{\"balance\":{\"lt\":%s}}}}", - TestsConstants.TEST_INDEX_ACCOUNT, ageToCompare, balanceToCompare)); - SearchHit[] hits = response.getHits(); - for (SearchHit hit : hits) { - int age = (int) hit.getSourceAsMap().get("age"); - int balance = (int) hit.getSourceAsMap().get("balance"); - assertThat(age, greaterThan(ageToCompare)); - assertThat(balance, lessThan(balanceToCompare)); - } - } - - @Test - /** - * Using TEST_INDEX_NESTED_TYPE here since term filter does not work properly on analyzed fields like text. - * The field 'someField' in TEST_INDEX_NESTED_TYPE is of type keyword. - */ - public void searchWithTermFilter() throws IOException { - /* - * Human readable format of the request defined below: - * { - * "query": "SELECT * FROM nested_objects WHERE nested(comment.likes) < 3", - * "filter": { - * "term": { - * "someField": "a" - * } - * } - * } - */ - int likesToCompare = 3; - String fieldToCompare = "a"; - SearchHits response = query(String.format("{\"query\":\"" + - "SELECT * " + - "FROM %s " + - "WHERE nested(comment.likes) < %s\"," + - "\"filter\":{\"term\":{\"someField\":\"%s\"}}}", - TestsConstants.TEST_INDEX_NESTED_TYPE, likesToCompare, fieldToCompare)); - SearchHit[] hits = response.getHits(); - for (SearchHit hit : hits) { - int likes = (int) ((Map) hit.getSourceAsMap().get("comment")).get("likes"); - String someField = hit.getSourceAsMap().get("someField").toString(); - assertThat(likes, lessThan(likesToCompare)); - assertThat(someField, equalTo(fieldToCompare)); - } - } - - @Test - public void searchWithNestedFilter() throws IOException { - /* - * Human readable format of the request defined below: - * { - * "query": "SELECT * FROM nested_objects WHERE nested(comment.likes) > 1", - * "filter": { - * "nested": { - * "path": "comment", - * "query": { - * "bool": { - * "must": { - * "term": { - * "comment.data": "aa" - * } - * } - * } - * } - * } - * } - * } - */ - int likesToCompare = 1; - String dataToCompare = "aa"; - SearchHits response = query(String.format("{\"query\":\"" + - "SELECT * " + - "FROM %s " + - "WHERE nested(comment.likes) > %s\"," + - "\"filter\":{\"nested\":{\"path\":\"comment\"," + - "\"query\":{\"bool\":{\"must\":{\"term\":{\"comment.data\":\"%s\"}}}}}}}", - TestsConstants.TEST_INDEX_NESTED_TYPE, likesToCompare, dataToCompare)); - SearchHit[] hits = response.getHits(); - for (SearchHit hit : hits) { - int likes = (int) ((Map) hit.getSourceAsMap().get("comment")).get("likes"); - String data = ((Map) hit.getSourceAsMap().get("comment")).get("data").toString(); - assertThat(likes, greaterThan(likesToCompare)); - assertThat(data, anyOf(equalTo(dataToCompare), equalTo("[aa, bb]"))); - } - } - - private SearchHits query(String request) throws IOException { - final JSONObject jsonObject = executeRequest(request); - - final XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser( - NamedXContentRegistry.EMPTY, - LoggingDeprecationHandler.INSTANCE, - jsonObject.toString()); - return SearchResponse.fromXContent(parser).getHits(); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/JdbcTestIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/JdbcTestIT.java deleted file mode 100644 index 3bdcf384fe..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/JdbcTestIT.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import org.json.JSONArray; -import org.json.JSONObject; -import org.junit.Assert; -import org.junit.Test; - -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; - -public class JdbcTestIT extends SQLIntegTestCase { - - @Override - protected void init() throws Exception { - loadIndex(Index.ONLINE); - loadIndex(Index.PEOPLE); - loadIndex(Index.ACCOUNT); - loadIndex(Index.WEBLOG); - } - - public void testPercentilesQuery() { - JSONObject response = executeJdbcRequest( - "SELECT percentiles(age, 25.0, 50.0, 75.0, 99.9) age_percentiles " + - "FROM elasticsearch-sql_test_index_people"); - - assertThat(response.getJSONArray("datarows").length(), equalTo(1)); - - JSONObject percentileRow = (JSONObject) response.query("/datarows/0/0"); - - assertThat(percentileRow.getDouble("25.0"), equalTo(31.5)); - assertThat(percentileRow.getDouble("50.0"), equalTo(33.5)); - assertThat(percentileRow.getDouble("75.0"), equalTo(36.5)); - assertThat(percentileRow.getDouble("99.9"), equalTo(39.0)); - } - - public void testDateTimeInQuery() { - JSONObject response = executeJdbcRequest( - "SELECT date_format(insert_time, 'dd-MM-YYYY') " + - "FROM elasticsearch-sql_test_index_online " + - "ORDER BY date_format(insert_time, 'dd-MM-YYYY') " + - "LIMIT 1" - ); - - assertThat( - response.getJSONArray("datarows") - .getJSONArray(0) - .getString(0), - equalTo("17-08-2014")); - } - - public void testDivisionInQuery() { - JSONObject response = executeJdbcRequest( - "SELECT all_client/10 from elasticsearch-sql_test_index_online ORDER BY all_client/10 desc limit 1"); - - assertThat( - response.getJSONArray("datarows") - .getJSONArray(0) - .getDouble(0), - equalTo(16827.0)); - } - - public void testGroupByInQuery() { - JSONObject response = executeJdbcRequest( - "SELECT date_format(insert_time, 'YYYY-MM-dd'), COUNT(*) " + - "FROM elasticsearch-sql_test_index_online " + - "GROUP BY date_format(insert_time, 'YYYY-MM-dd')" - ); - - assertThat(response.getJSONArray("schema").length(), equalTo(2)); - assertThat(response.getJSONArray("datarows").length(), equalTo(8)); - } - - private JSONObject executeJdbcRequest(String query){ - return new JSONObject(executeQuery(query, "jdbc")); - } - - @Test - public void numberOperatorNameCaseInsensitiveTest() { - assertSchemaContains( - executeQuery("SELECT ABS(age) FROM elasticsearch-sql_test_index_account " + - "WHERE age IS NOT NULL ORDER BY age LIMIT 5", "jdbc"), - "ABS(age)" - ); - } - - @Test - public void trigFunctionNameCaseInsensitiveTest() { - assertSchemaContains( - executeQuery("SELECT Cos(age) FROM elasticsearch-sql_test_index_account " + - "WHERE age is NOT NULL ORDER BY age LIMIT 5", "jdbc"), - "Cos(age)" - ); - } - - @Test - public void stringOperatorNameCaseInsensitiveTest() { - assertSchemaContains( - executeQuery("SELECT SubStrinG(lastname, 0, 2) FROM elasticsearch-sql_test_index_account " + - "ORDER BY age LIMIT 5", "jdbc"), - "SubStrinG(lastname, 0, 2)" - ); - } - - @Test - public void dateFunctionNameCaseInsensitiveTest() { - assertTrue( - executeQuery("SELECT DATE_FORMAT(insert_time, 'yyyy-MM-dd', 'UTC') FROM elasticsearch-sql_test_index_online " + - "WHERE date_FORMAT(insert_time, 'yyyy-MM-dd', 'UTC') > '2014-01-01' " + - "GROUP BY DAte_format(insert_time, 'yyyy-MM-dd', 'UTC') " + - "ORDER BY date_forMAT(insert_time, 'yyyy-MM-dd', 'UTC')", "jdbc").equalsIgnoreCase( - executeQuery("SELECT date_format(insert_time, 'yyyy-MM-dd', 'UTC') FROM elasticsearch-sql_test_index_online " + - "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') > '2014-01-01' " + - "GROUP BY date_format(insert_time, 'yyyy-MM-dd', 'UTC') " + - "ORDER BY date_format(insert_time, 'yyyy-MM-dd', 'UTC')", "jdbc") - ) - ); - } - - @Test - public void ipTypeShouldPassJdbcFormatter() { - assertThat( - executeQuery("SELECT host AS hostIP FROM " + TestsConstants.TEST_INDEX_WEBLOG - + " ORDER BY hostIP", "jdbc"), - containsString("\"type\": \"ip\"") - ); - } - - @Test - public void functionWithoutAliasShouldHaveEntireFunctionAsNameInSchema() { - assertThat( - executeQuery("SELECT substring(lastname, 1, 2) FROM " + TestsConstants.TEST_INDEX_ACCOUNT - + " ORDER BY substring(lastname, 1, 2)", "jdbc"), - containsString("\"name\": \"substring(lastname, 1, 2)\"") - ); - } - - @Test - public void functionWithAliasShouldHaveAliasAsNameInSchema() { - assertThat( - executeQuery("SELECT substring(lastname, 1, 2) AS substring FROM " - + TestsConstants.TEST_INDEX_ACCOUNT + " ORDER BY substring", "jdbc"), - containsString("\"name\": \"substring\"") - ); - } - - private void assertSchemaContains(String actualResponse, String expected) { - JSONArray schema = new JSONObject(actualResponse).optJSONArray("schema"); - for (Object nameTypePair : schema) { - String actual = ((JSONObject) nameTypePair).getString("name"); - if (expected.equals(actual)) { - return; - } - } - Assert.fail("Expected field name [" + expected + "] is not found in response schema: " + actualResponse); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/JoinAliasWriterRuleIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/JoinAliasWriterRuleIT.java deleted file mode 100644 index 8e6c6f2bb1..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/JoinAliasWriterRuleIT.java +++ /dev/null @@ -1,261 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import org.elasticsearch.client.ResponseException; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import java.io.IOException; - -import static org.hamcrest.Matchers.equalTo; - -/** - * Test cases for writing missing join table aliases. - */ -public class JoinAliasWriterRuleIT extends SQLIntegTestCase { - - @Rule - public ExpectedException exception = ExpectedException.none(); - - protected void init() throws Exception { - loadIndex(Index.ORDER); // elasticsearch-sql_test_index_order - loadIndex(Index.BANK); // elasticsearch-sql_test_index_bank - loadIndex(Index.BANK_TWO); // elasticsearch-sql_test_index_bank_two - } - - @Test - public void noTableAliasNoCommonColumns() throws IOException { - sameExplain( - query( - "SELECT id, firstname" , - "FROM elasticsearch-sql_test_index_order", - "INNER JOIN elasticsearch-sql_test_index_bank ", - "ON name = firstname WHERE state = 'WA' OR id < 7"), - query( - "SELECT elasticsearch-sql_test_index_order_0.id, elasticsearch-sql_test_index_bank_1.firstname ", - "FROM elasticsearch-sql_test_index_order elasticsearch-sql_test_index_order_0 ", - "INNER JOIN elasticsearch-sql_test_index_bank elasticsearch-sql_test_index_bank_1 " , - "ON elasticsearch-sql_test_index_order_0.name = elasticsearch-sql_test_index_bank_1.firstname ", - "WHERE elasticsearch-sql_test_index_bank_1.state = 'WA' OR elasticsearch-sql_test_index_order_0.id < 7") - ); - } - - @Test - public void oneTableAliasNoCommonColumns() throws IOException { - sameExplain( - query( - "SELECT id, firstname ", - "FROM elasticsearch-sql_test_index_order a ", - "INNER JOIN elasticsearch-sql_test_index_bank ", - "ON name = firstname WHERE state = 'WA' OR id < 7"), - query( - "SELECT a.id, elasticsearch-sql_test_index_bank_0.firstname ", - "FROM elasticsearch-sql_test_index_order a ", - "INNER JOIN elasticsearch-sql_test_index_bank elasticsearch-sql_test_index_bank_0 ", - "ON a.name = elasticsearch-sql_test_index_bank_0.firstname ", - "WHERE elasticsearch-sql_test_index_bank_0.state = 'WA' OR a.id < 7") - ); - } - - @Test - public void bothTableAliasNoCommonColumns() throws IOException { - sameExplain( - query( - "SELECT id, firstname ", - "FROM elasticsearch-sql_test_index_order a ", - "INNER JOIN elasticsearch-sql_test_index_bank b ", - "ON name = firstname WHERE state = 'WA' OR id < 7 "), - query( - "SELECT a.id, b.firstname ", - "FROM elasticsearch-sql_test_index_order a ", - "INNER JOIN elasticsearch-sql_test_index_bank b ", - "ON a.name = b.firstname ", - "WHERE b.state = 'WA' OR a.id < 7 ") - ); - } - - @Test - public void tableNamesWithTypeName() throws IOException { - sameExplain( - query( - "SELECT id, firstname ", - "FROM elasticsearch-sql_test_index_order/_doc ", - "INNER JOIN elasticsearch-sql_test_index_bank/account ", - "ON name = firstname WHERE state = 'WA' OR id < 7"), - query( - "SELECT elasticsearch-sql_test_index_order_0.id, elasticsearch-sql_test_index_bank_1.firstname ", - "FROM elasticsearch-sql_test_index_order/_doc elasticsearch-sql_test_index_order_0 ", - "INNER JOIN elasticsearch-sql_test_index_bank/_account elasticsearch-sql_test_index_bank_1 ", - "ON elasticsearch-sql_test_index_order_0.name = elasticsearch-sql_test_index_bank_1.firstname ", - "WHERE elasticsearch-sql_test_index_bank_1.state = 'WA' OR elasticsearch-sql_test_index_order_0.id < 7") - ); - } - - @Ignore - @Test - public void tableNamesWithTypeNameExplicitTableAlias() throws IOException { - sameExplain( - query( - "SELECT id, firstname " , - "FROM elasticsearch-sql_test_index_order/_doc a " , - "INNER JOIN elasticsearch-sql_test_index_bank/account b ", - "ON name = firstname WHERE state = 'WA' OR id < 7"), - query( - "SELECT a.id, b.firstname ", - "FROM elasticsearch-sql_test_index_order a ", - "INNER JOIN elasticsearch-sql_test_index_bank b " , - "ON a.name = b.firstname ", - "WHERE b.state = 'WA' OR a.id < 7") - ); - } - - @Test - public void actualTableNameAsAliasOnColumnFields() throws IOException { - sameExplain( - query( - "SELECT elasticsearch-sql_test_index_order.id, b.firstname " , - "FROM elasticsearch-sql_test_index_order " , - "INNER JOIN elasticsearch-sql_test_index_bank b ", - "ON elasticsearch-sql_test_index_order.name = firstname WHERE state = 'WA' OR id < 7"), - query( - "SELECT elasticsearch-sql_test_index_order_0.id, b.firstname ", - "FROM elasticsearch-sql_test_index_order elasticsearch-sql_test_index_order_0 ", - "INNER JOIN elasticsearch-sql_test_index_bank b " , - "ON elasticsearch-sql_test_index_order_0.name = b.firstname ", - "WHERE b.state = 'WA' OR elasticsearch-sql_test_index_order_0.id < 7") - ); - } - - @Test - public void actualTableNameAsAliasOnColumnFieldsTwo() throws IOException { - sameExplain( - query( - "SELECT elasticsearch-sql_test_index_order.id, elasticsearch-sql_test_index_bank.firstname " , - "FROM elasticsearch-sql_test_index_order " , - "INNER JOIN elasticsearch-sql_test_index_bank ", - "ON elasticsearch-sql_test_index_order.name = firstname ", - "WHERE elasticsearch-sql_test_index_bank.state = 'WA' OR id < 7"), - query( - "SELECT elasticsearch-sql_test_index_order_0.id, elasticsearch-sql_test_index_bank_1.firstname ", - "FROM elasticsearch-sql_test_index_order elasticsearch-sql_test_index_order_0 ", - "INNER JOIN elasticsearch-sql_test_index_bank elasticsearch-sql_test_index_bank_1" , - "ON elasticsearch-sql_test_index_order_0.name = elasticsearch-sql_test_index_bank_1.firstname ", - "WHERE elasticsearch-sql_test_index_bank_1.state = 'WA' OR elasticsearch-sql_test_index_order_0.id < 7") - ); - } - - @Test - public void columnsWithTableAliasNotAffected() throws IOException { - sameExplain( - query( - "SELECT a.id, firstname ", - "FROM elasticsearch-sql_test_index_order a ", - "INNER JOIN elasticsearch-sql_test_index_bank b ", - "ON name = b.firstname WHERE state = 'WA' OR a.id < 7"), - query( - "SELECT a.id, b.firstname ", - "FROM elasticsearch-sql_test_index_order a ", - "INNER JOIN elasticsearch-sql_test_index_bank b ", - "ON a.name = b.firstname ", - "WHERE b.state = 'WA' OR a.id < 7") - ); - } - - @Test - public void commonColumnWithoutTableAliasDifferentTables() throws IOException { - exception.expect(ResponseException.class); - exception.expectMessage("Field name [firstname] is ambiguous"); - String explain = explainQuery(query( - "SELECT firstname, lastname ", - "FROM elasticsearch-sql_test_index_bank ", - "LEFT JOIN elasticsearch-sql_test_index_bank_two ", - "ON firstname = lastname WHERE state = 'VA' " - )); - } - - @Test - public void sameTablesNoAliasAndNoAliasOnColumns() throws IOException { - exception.expect(ResponseException.class); - exception.expectMessage("Not unique table/alias: [elasticsearch-sql_test_index_bank]"); - String explain = explainQuery(query( - "SELECT firstname, lastname ", - "FROM elasticsearch-sql_test_index_bank ", - "LEFT JOIN elasticsearch-sql_test_index_bank ", - "ON firstname = lastname WHERE state = 'VA' " - )); - } - - @Test - public void sameTablesNoAliasWithTableNameAsAliasOnColumns() throws IOException { - exception.expect(ResponseException.class); - exception.expectMessage("Not unique table/alias: [elasticsearch-sql_test_index_bank]"); - String explain = explainQuery(query( - "SELECT elasticsearch-sql_test_index_bank.firstname", - "FROM elasticsearch-sql_test_index_bank ", - "JOIN elasticsearch-sql_test_index_bank ", - "ON elasticsearch-sql_test_index_bank.firstname = elasticsearch-sql_test_index_bank.lastname" - )); - } - - @Test - public void sameTablesWithExplicitAliasOnFirst() throws IOException { - sameExplain( - query( - "SELECT elasticsearch-sql_test_index_bank.firstname, a.lastname ", - "FROM elasticsearch-sql_test_index_bank a", - "JOIN elasticsearch-sql_test_index_bank ", - "ON elasticsearch-sql_test_index_bank.firstname = a.lastname " - ), - query( - "SELECT elasticsearch-sql_test_index_bank_0.firstname, a.lastname ", - "FROM elasticsearch-sql_test_index_bank a", - "JOIN elasticsearch-sql_test_index_bank elasticsearch-sql_test_index_bank_0", - "ON elasticsearch-sql_test_index_bank_0.firstname = a.lastname " - ) - - ); - } - - @Test - public void sameTablesWithExplicitAliasOnSecond() throws IOException { - sameExplain( - query( - "SELECT elasticsearch-sql_test_index_bank.firstname, a.lastname ", - "FROM elasticsearch-sql_test_index_bank ", - "JOIN elasticsearch-sql_test_index_bank a", - "ON elasticsearch-sql_test_index_bank.firstname = a.lastname " - ), - query( - "SELECT elasticsearch-sql_test_index_bank_0.firstname, a.lastname ", - "FROM elasticsearch-sql_test_index_bank elasticsearch-sql_test_index_bank_0", - "JOIN elasticsearch-sql_test_index_bank a", - "ON elasticsearch-sql_test_index_bank_0.firstname = a.lastname " - ) - - ); - } - - private void sameExplain(String actualQuery, String expectedQuery) throws IOException { - assertThat(explainQuery(actualQuery), equalTo(explainQuery(expectedQuery))); - } - - private String query(String... statements) { - return String.join(" ", statements); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/JoinIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/JoinIT.java deleted file mode 100644 index 34ab073b49..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/JoinIT.java +++ /dev/null @@ -1,1010 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import com.google.common.collect.ImmutableMap; -import org.json.JSONArray; -import org.json.JSONObject; -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; -import java.util.regex.Pattern; -import java.util.stream.IntStream; - -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_ACCOUNT; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_DOG; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_DOG2; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_GAME_OF_THRONES; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_LOCATION; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_LOCATION2; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_PEOPLE; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_PEOPLE2; -import static org.hamcrest.Matchers.anyOf; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; - -public class JoinIT extends SQLIntegTestCase { - - private static final String USE_NL_HINT = " /*! USE_NL*/"; - - @Override - protected void init() throws Exception { - - loadIndex(Index.DOG); - loadIndex(Index.DOGS2); - loadIndex(Index.PEOPLE); - loadIndex(Index.PEOPLE2); - loadIndex(Index.GAME_OF_THRONES); - loadIndex(Index.LOCATION); - loadIndex(Index.LOCATION_TWO); - loadIndex(Index.ACCOUNT); - } - - @Test - public void joinParseCheckSelectedFieldsSplitHASH() throws IOException { - joinParseCheckSelectedFieldsSplit(false); - } - - @Test - public void joinParseCheckSelectedFieldsSplitNL() throws IOException { - joinParseCheckSelectedFieldsSplit(true); - } - - @Test - public void joinParseWithHintsCheckSelectedFieldsSplitHASH() throws IOException { - - String query = String.format(Locale.ROOT, "SELECT /*! HASH_WITH_TERMS_FILTER*/ " + - "a.firstname ,a.lastname, a.gender ,d.dog_name FROM %s a JOIN %s d " + - "ON d.holdersName = a.firstname WHERE (a.age > 10 OR a.balance > 2000) AND d.age > 1", - TEST_INDEX_PEOPLE, TEST_INDEX_DOG); - - JSONObject result = executeQuery(query); - verifyJoinParseCheckSelectedFieldsSplitResult(result, false); - - String explanation = explainQuery(query); - Assert.assertThat(explanation, containsString("holdersName")); - - // TODO: figure out why explain does not show results from first query in term filter and - // fix either the test or the code. - //Arrays.asList("daenerys","nanette","virginia","aurelia","mcgee","hattie","elinor","burton").forEach(name -> { - // Assert.assertThat(explanation, containsString(name)); - //}); - } - - @Test - public void joinWithNoWhereButWithConditionHash() throws IOException { - - joinWithNoWhereButWithCondition(false); - } - - @Test - public void joinWithNoWhereButWithConditionNL() throws IOException { - - joinWithNoWhereButWithCondition(true); - } - - @Test - public void joinWithStarHASH() throws IOException { - - String query = String.format(Locale.ROOT, "SELECT * FROM %1$s c " + - "JOIN %1$s h ON h.hname = c.house ", TEST_INDEX_GAME_OF_THRONES); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - - Assert.assertEquals(4, hits.length()); - String house = (String)hits.query("/0/_source/c.house"); - Assert.assertThat(house, anyOf(equalTo("Targaryen"), equalTo("Stark"), equalTo("Lannister"))); - String houseName = (String)hits.query("/0/_source/h.hname"); - Assert.assertEquals(house, houseName); - } - - @Test - public void joinNoConditionButWithWhereHASH() throws IOException { - - joinNoConditionButWithWhere(false); - } - - @Test - public void joinNoConditionButWithWhereNL() throws IOException { - - joinNoConditionButWithWhere(true); - } - - @Test - public void joinNoConditionAndNoWhereHASH() throws IOException { - - joinNoConditionAndNoWhere(false); - } - - @Test - public void joinNoConditionAndNoWhereNL() throws IOException { - - joinNoConditionAndNoWhere(true); - } - - @Test - public void joinNoConditionAndNoWhereWithTotalLimitHASH() throws IOException { - - joinNoConditionAndNoWhereWithTotalLimit(false); - } - - @Test - public void joinNoConditionAndNoWhereWithTotalLimitNL() throws IOException { - - joinNoConditionAndNoWhereWithTotalLimit(true); - } - - @Test - public void joinWithNestedFieldsOnReturnHASH() throws IOException { - - joinWithNestedFieldsOnReturn(false); - } - - @Test - public void joinWithNestedFieldsOnReturnNL() throws IOException { - - joinWithNestedFieldsOnReturn(true); - } - - @Test - public void joinWithAllAliasOnReturnHASH() throws IOException { - - joinWithAllAliasOnReturn(false); - } - @Test - public void joinWithAllAliasOnReturnNL() throws IOException { - - joinWithAllAliasOnReturn(true); - } - - @Test - public void joinWithSomeAliasOnReturnHASH() throws IOException { - - joinWithSomeAliasOnReturn(false); - } - - @Test - public void joinWithSomeAliasOnReturnNL() throws IOException { - - joinWithSomeAliasOnReturn(true); - } - - @Test - public void joinWithNestedFieldsOnComparisonAndOnReturnHASH() throws IOException { - - joinWithNestedFieldsOnComparisonAndOnReturn(false); - } - - @Test - public void joinWithNestedFieldsOnComparisonAndOnReturnNL() throws IOException { - - joinWithNestedFieldsOnComparisonAndOnReturn(true); - } - - @Test - public void testLeftJoinHASH() throws IOException { - - testLeftJoin(false); - } - - @Test - public void testLeftJoinNL() throws IOException { - - testLeftJoin(true); - } - - @Test - public void hintLimits_firstLimitSecondNullHASH() throws IOException { - - hintLimits_firstLimitSecondNull(false); - } - - @Test - public void hintLimits_firstLimitSecondNullNL() throws IOException { - - hintLimits_firstLimitSecondNull(true); - } - - @Test - public void hintLimits_firstLimitSecondLimitHASH() throws IOException { - - hintLimits_firstLimitSecondLimit(false); - } - - @Test - public void hintLimits_firstLimitSecondLimitNL() throws IOException { - - hintLimits_firstLimitSecondLimit(true); - } - - @Ignore("Join limit hint is deprecated and easily broken due to limit on unsorted records") - @Test - public void hintLimits_firstLimitSecondLimitOnlyOneNL() throws IOException { - - hintLimits_firstLimitSecondLimitOnlyOne(true); - } - - @Ignore("Join limit hint is deprecated and easily broken due to limit on unsorted records") - @Test - public void hintLimits_firstLimitSecondLimitOnlyOneHASH() throws IOException { - - hintLimits_firstLimitSecondLimitOnlyOne(false); - } - - @Test - public void hintLimits_firstNullSecondLimitHASH() throws IOException { - - hintLimits_firstNullSecondLimit(false); - } - - @Test - public void hintLimits_firstNullSecondLimitNL() throws IOException { - - hintLimits_firstNullSecondLimit(true); - } - - @Test - public void testLeftJoinWithLimitHASH() throws IOException { - - testLeftJoinWithLimit(false); - } - - @Test - public void testLeftJoinWithLimitNL() throws IOException { - - testLeftJoinWithLimit(true); - } - - @Test - public void hintMultiSearchCanRunFewTimesNL() throws IOException { - - String query = String.format(Locale.ROOT, "SELECT /*! USE_NL*/ /*! NL_MULTISEARCH_SIZE(2)*/ " + - "c.name.firstname,c.parents.father,h.hname,h.words FROM %1$s c " + - "JOIN %1$s h", TEST_INDEX_GAME_OF_THRONES); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - Assert.assertThat(hits.length(), equalTo(42)); - } - - @Test - public void joinWithGeoIntersectNL() throws IOException { - - String query = String.format(Locale.ROOT, "SELECT p1.description,p2.description " + - "FROM %s p1 JOIN %s p2 ON GEO_INTERSECTS(p2.place,p1.place)", - TEST_INDEX_LOCATION, TEST_INDEX_LOCATION2); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - Assert.assertThat(hits.length(), equalTo(2)); - - Assert.assertThat(hits.query("/0/_source/p2.description"), equalTo("squareRelated")); - Assert.assertThat(hits.query("/1/_source/p2.description"), equalTo("squareRelated")); - } - - // TODO: resolve issue #25 in github and update the test if needed - // TODO: this test causes the in-memory node to fail/crash - @Ignore - @Test - public void joinWithInQuery() throws IOException { - - //TODO: Either change the ON condition field to keyword or create a different subquery - String query = String.format(Locale.ROOT, "SELECT c.gender,c.name.firstname,h.hname,h.words " + - "FROM %1$s c JOIN %1$s h ON h.hname = c.house " + - "WHERE c.name.firstname IN (SELECT holdersName FROM %2$s)", - TEST_INDEX_GAME_OF_THRONES, TEST_INDEX_DOG); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - Assert.assertThat(hits.length(), equalTo(1)); - Assert.assertThat(hits.query("/0/_source/c.name.firstname"), equalTo("Daenerys")); - } - - @Test - public void joinWithOrHASH() throws IOException { - - joinWithOr(false); - } - - @Test - public void joinWithOrNL() throws IOException { - - joinWithOr(true); - } - - @Ignore // TODO: explanation does not have the terms section - @Test - public void joinWithOrWithTermsFilterOpt() throws IOException { - - String query = String.format(Locale.ROOT, "SELECT /*! HASH_WITH_TERMS_FILTER*/ " + - "d.dog_name,c.name.firstname FROM %s c " + - "JOIN %s d ON d.holdersName = c.name.firstname OR d.age = c.name.ofHisName", - TEST_INDEX_GAME_OF_THRONES, TEST_INDEX_DOG); - - executeQuery(query); - String explanation = explainQuery(query); - - Assert.assertTrue(containsTerm(explanation, "holdersName")); - Assert.assertTrue(containsTerm(explanation, "age")); - - Arrays.asList("daenerys", "brandon", "eddard", "jaime").forEach( - name -> Assert.assertTrue(explanation.contains(name)) - ); - } - - @Test - public void joinWithOrderbyFirstTableHASH() throws IOException { - - joinWithOrderFirstTable(false); - } - - @Test - public void joinWithOrderbyFirstTableNL() throws IOException { - - joinWithOrderFirstTable(true); - } - - @Test - public void joinWithAllFromSecondTableHASH() throws IOException { - - joinWithAllFromSecondTable(false); - } - - @Test - public void joinWithAllFromSecondTableNL() throws IOException { - - joinWithAllFromSecondTable(true); - } - - @Test - public void joinWithAllFromFirstTableHASH() throws IOException { - - joinWithAllFromFirstTable(false); - } - - @Test - public void joinWithAllFromFirstTableNL() throws IOException { - - joinWithAllFromFirstTable(true); - } - - @Test - public void leftJoinWithAllFromSecondTableHASH() throws IOException { - - leftJoinWithAllFromSecondTable(false); - } - - @Test - public void leftJoinWithAllFromSecondTableNL() throws IOException { - - leftJoinWithAllFromSecondTable(true); - } - - @Test - public void joinParseCheckSelectedFieldsSplitNLConditionOrderEQ() throws IOException { - - final String query = String.format(Locale.ROOT, "SELECT /*! USE_NL*/ " + - "a.firstname, a.lastname, a.gender, d.dog_name FROM %s a JOIN %s d " + - "ON a.firstname = d.holdersName WHERE (a.age > 10 OR a.balance > 2000) AND d.age > 1", - TEST_INDEX_PEOPLE2, TEST_INDEX_DOG2); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - - Assert.assertThat(hits.length(), equalTo(2)); - - Map match1 = ImmutableMap.of( - "a.firstname", "Daenerys", - "a.lastname", "Targaryen", - "a.gender", "M", - "d.dog_name", "rex"); - Map match2 = ImmutableMap.of( - "a.firstname", "Hattie", - "a.lastname", "Bond", - "a.gender", "M", - "d.dog_name", "snoopy"); - - Assert.assertTrue(hitsInclude(hits, match1)); - Assert.assertTrue(hitsInclude(hits, match2)); - } - - @Test - public void joinParseCheckSelectedFieldsSplitNLConditionOrderGT() throws IOException { - - final String query = String.format(Locale.ROOT, "SELECT /*! USE_NL*/ " + - "a.firstname, a.lastname, a.gender, d.firstname, d.age FROM " + - "%s a JOIN %s d on a.age < d.age " + - "WHERE (d.firstname = 'Lynn' OR d.firstname = 'Obrien') AND a.firstname = 'Mcgee'", - TEST_INDEX_PEOPLE, TEST_INDEX_ACCOUNT); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - - Assert.assertThat(hits.length(), equalTo(2)); - - Map oneMatch = ImmutableMap.of("a.firstname", "Mcgee", "a.lastname", "Mooney", - "a.gender", "M", "d.firstname", "Obrien", "d.age", 40); - Map secondMatch = ImmutableMap.of("a.firstname", "Mcgee", "a.lastname", "Mooney", - "a.gender", "M", "d.firstname", "Lynn", "d.age", 40); - - Assert.assertTrue(hitsInclude(hits, oneMatch)); - Assert.assertTrue(hitsInclude(hits, secondMatch)); - } - - @Test - public void joinParseCheckSelectedFieldsSplitNLConditionOrderLT() throws IOException { - - final String query = String.format(Locale.ROOT, "SELECT /*! USE_NL*/ " + - "a.firstname, a.lastname, a.gender, d.firstname, d.age FROM " + - "%s a JOIN %s d on a.age > d.age " + - "WHERE (d.firstname = 'Sandoval' OR d.firstname = 'Hewitt') AND a.firstname = 'Fulton'", - TEST_INDEX_PEOPLE, TEST_INDEX_ACCOUNT); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - - Assert.assertThat(hits.length(), equalTo(2)); - - Map oneMatch = ImmutableMap.of("a.firstname", "Fulton", "a.lastname", "Holt", - "a.gender", "F", "d.firstname", "Sandoval", "d.age", 22); - Map secondMatch = ImmutableMap.of("a.firstname", "Fulton", "a.lastname", "Holt", - "a.gender", "F", "d.firstname", "Hewitt", "d.age", 22); - - Assert.assertTrue(hitsInclude(hits, oneMatch)); - Assert.assertTrue(hitsInclude(hits, secondMatch)); - } - - @Test - public void leftJoinNLWithNullInCondition() throws IOException { - - joinWithNullInCondition(true, "LEFT", "OR", "OR", 7); - } - - @Test - public void leftJoinNLWithNullInCondition1() throws IOException { - - joinWithNullInCondition(true, "LEFT", "OR", "AND", 7); - } - - @Test - public void leftJoinNLWithNullInCondition2() throws IOException { - - joinWithNullInCondition(true, "LEFT", "AND", "AND", 7); - } - - @Test - public void leftJoinNLWithNullInCondition3() throws IOException { - - joinWithNullInCondition(true, "LEFT", "AND", "OR", 7); - } - - @Test - public void innerJoinNLWithNullInCondition() throws IOException { - - joinWithNullInCondition(true, "", "OR", "OR", 0 ); - } - - @Test - public void innerJoinNLWithNullInCondition1() throws IOException { - - joinWithNullInCondition(true, "", "OR", "AND", 0); - } - - @Test - public void innerJoinNLWithNullInCondition2() throws IOException { - - joinWithNullInCondition(true, "", "AND", "AND", 0); - } - - @Test - public void innerJoinNLWithNullInCondition3() throws IOException { - - joinWithNullInCondition(true, "", "AND", "OR", 0); - } - - private void joinWithAllFromSecondTable(boolean useNestedLoops) throws IOException { - - final String hint = useNestedLoops ? USE_NL_HINT : ""; - final String query = String.format(Locale.ROOT, "SELECT%1$s c.name.firstname, d.* " + - "FROM %2$s c JOIN %2$s d ON d.hname = c.house", - hint, TEST_INDEX_GAME_OF_THRONES); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - - if (useNestedLoops) { - Assert.assertThat(hits.length(), equalTo(0)); - } else { - Assert.assertThat(hits.length(), equalTo(4)); - Assert.assertThat(hits.getJSONObject(0).getJSONObject("_source").length(), equalTo(5)); - } - } - - private void joinWithAllFromFirstTable(boolean useNestedLoops) throws IOException { - - final String hint = useNestedLoops ? USE_NL_HINT : ""; - final String query = String.format(Locale.ROOT, "SELECT%1$s c.name.firstname " + - "FROM %2$s d JOIN %2$s c ON c.house = d.hname", - hint, TEST_INDEX_GAME_OF_THRONES); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - - if (useNestedLoops) { - Assert.assertThat(hits.length(), equalTo(0)); - } else { - Assert.assertThat(hits.length(), equalTo(4)); - Assert.assertThat(hits.getJSONObject(0).getJSONObject("_source").length(), equalTo(1)); - } - } - - private void leftJoinWithAllFromSecondTable(boolean useNestedLoops) throws IOException { - - final String hint = useNestedLoops ? USE_NL_HINT : ""; - final String query = String.format(Locale.ROOT, "SELECT%1$s c.name.firstname, d.* " + - "FROM %2$s c LEFT JOIN %2$s d ON d.hname = c.house", - hint, TEST_INDEX_GAME_OF_THRONES); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - - Assert.assertThat(hits.length(), equalTo(7)); - - hits.forEach(hitObj -> { - JSONObject hit = (JSONObject)hitObj; - - Assert.assertThat(hit.getJSONObject("_source").length(), - equalTo(hit.getString("_id").endsWith("0") ? 1 : 5)); - }); - } - - private void joinParseCheckSelectedFieldsSplit(boolean useNestedLoops) throws IOException { - - final String hint = useNestedLoops ? USE_NL_HINT : ""; - String query = String.format(Locale.ROOT, "SELECT%s a.firstname ,a.lastname,a.gender,d.dog_name " + - "FROM %s a JOIN %s d ON d.holdersName = a.firstname " + - "WHERE (a.age > 10 OR a.balance > 2000) AND d.age > 1", hint, TEST_INDEX_PEOPLE, TEST_INDEX_DOG); - - JSONObject result = executeQuery(query); - verifyJoinParseCheckSelectedFieldsSplitResult(result, useNestedLoops); - } - - private void joinNoConditionButWithWhere(boolean useNestedLoops) throws IOException { - - final String hint = useNestedLoops ? USE_NL_HINT : ""; - String query = String.format(Locale.ROOT, "SELECT%s c.gender,h.hname,h.words FROM %2$s c " + - "JOIN %2$s h WHERE match_phrase(c.name.firstname, 'Daenerys')", - hint, TEST_INDEX_GAME_OF_THRONES); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - Assert.assertThat(hits.length(), equalTo(7)); - } - - private void joinNoConditionAndNoWhere(boolean useNestedLoops) throws IOException { - - final String hint = useNestedLoops ? USE_NL_HINT : ""; - String query = String.format(Locale.ROOT, "SELECT%s c.name.firstname,c.parents.father,h.hname,h.words " + - "FROM %2$s c JOIN %2$s h", - hint, TEST_INDEX_GAME_OF_THRONES); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - Assert.assertThat(hits.length(), equalTo(49)); - } - - private void joinWithNoWhereButWithCondition(boolean useNestedLoops) throws IOException { - - final String hint = useNestedLoops ? USE_NL_HINT : ""; - String query = String.format(Locale.ROOT, "SELECT%s c.gender,h.hname,h.words " + - "FROM %2$s c JOIN %2$s h ON h.hname = c.house", - hint, TEST_INDEX_GAME_OF_THRONES); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - - Map someMatch = ImmutableMap.of( - "c.gender", "F", - "h.hname","Targaryen", - "h.words", "fireAndBlood"); - - if (useNestedLoops) { - // TODO: should the NL result be different? - Assert.assertThat(hits.length(), equalTo(0)); - } else { - Assert.assertThat(hits.length(), equalTo(4)); - Assert.assertTrue(hitsInclude(hits, someMatch)); - } - } - - private void verifyJoinParseCheckSelectedFieldsSplitResult(JSONObject result, boolean useNestedLoops) { - - Map match1 = ImmutableMap.of( - "a.firstname", "Daenerys", - "a.lastname", "Targaryen", - "a.gender", "M", - "d.dog_name", "rex"); - Map match2 = ImmutableMap.of( - "a.firstname", "Hattie", - "a.lastname", "Bond", - "a.gender", "M", - "d.dog_name", "snoopy"); - - JSONArray hits = getHits(result); - - if (useNestedLoops) { - //TODO: change field mapping in ON condition to keyword or change query to get result - // TODO: why does NL query return no results? - Assert.assertThat(hits.length(), equalTo(0)); - } else { - Assert.assertThat(hits.length(), equalTo(2)); - Assert.assertTrue(hitsInclude(hits, match1)); - Assert.assertTrue(hitsInclude(hits, match2)); - } - } - - private void joinNoConditionAndNoWhereWithTotalLimit(boolean useNestedLoops) throws IOException { - - final String hint = useNestedLoops ? USE_NL_HINT : ""; - String query = String.format(Locale.ROOT, "SELECT%s c.name.firstname,c.parents.father,h.hname,h.words" + - " FROM %2$s c JOIN %2$s h LIMIT 9", - hint, TEST_INDEX_GAME_OF_THRONES); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - Assert.assertThat(hits.length(), equalTo(9)); - } - - private void joinWithNestedFieldsOnReturn(boolean useNestedLoops) throws IOException { - - final String hint = useNestedLoops ? USE_NL_HINT : ""; - String query = String.format(Locale.ROOT, "SELECT%s c.name.firstname,c.parents.father,h.hname,h.words " + - "FROM %2$s c JOIN %2$s h ON h.hname = c.house " + - "WHERE match_phrase(c.name.firstname, 'Daenerys')", - hint, TEST_INDEX_GAME_OF_THRONES); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - final Map expectedMatch = ImmutableMap.of( - "c.name.firstname", "Daenerys", - "c.parents.father", "Aerys", - "h.hname", "Targaryen", - "h.words", "fireAndBlood"); - if (useNestedLoops) { - Assert.assertThat(hits.length(), equalTo(0)); - } else { - Assert.assertThat(hits.length(), equalTo(1)); - assertHitMatches(hits.getJSONObject(0), expectedMatch); - } - } - - private void joinWithAllAliasOnReturn(boolean useNestedLoops) throws IOException { - - final String hint = useNestedLoops ? USE_NL_HINT : ""; - String query = String.format(Locale.ROOT, "SELECT%s c.name.firstname name,c.parents.father father," + - "h.hname house FROM %2$s c JOIN %2$s h ON h.hname = c.house " + - "WHERE match_phrase(c.name.firstname, 'Daenerys')", - hint, TEST_INDEX_GAME_OF_THRONES); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - final Map expectedMatch = ImmutableMap.of( - "name", "Daenerys", - "father", "Aerys", - "house", "Targaryen"); - - if (useNestedLoops) { - Assert.assertThat(hits.length(), equalTo(0)); - } else { - Assert.assertThat(hits.length(), equalTo(1)); - assertHitMatches(hits.getJSONObject(0), expectedMatch); - } - } - - private void joinWithSomeAliasOnReturn(boolean useNestedLoops) throws IOException { - - final String hint = useNestedLoops ? USE_NL_HINT : ""; - String query = String.format(Locale.ROOT, "SELECT%s c.name.firstname ,c.parents.father father, " + - "h.hname house FROM %2$s c JOIN %2$s h ON h.hname = c.house " + - "WHERE match_phrase(c.name.firstname, 'Daenerys')", - hint, TEST_INDEX_GAME_OF_THRONES); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - final Map expectedMatch = ImmutableMap.of( - "c.name.firstname", "Daenerys", - "father", "Aerys", - "house", "Targaryen"); - - if (useNestedLoops) { - //TODO: Either change the ON condition field to keyword or create a different subquery - Assert.assertThat(hits.length(), equalTo(0)); - } else { - Assert.assertThat(hits.length(), equalTo(1)); - assertHitMatches(hits.getJSONObject(0), expectedMatch); - } - } - - private void joinWithNestedFieldsOnComparisonAndOnReturn(boolean useNestedLoops) throws IOException { - - final String hint = useNestedLoops ? USE_NL_HINT : ""; - String query = String.format(Locale.ROOT, "SELECT%s c.name.firstname,c.parents.father, h.hname,h.words " + - " FROM %2$s c JOIN %2$s h ON h.hname = c.name.lastname " + - "WHERE match_phrase(c.name.firstname, 'Daenerys')", - hint, TEST_INDEX_GAME_OF_THRONES); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - final Map expectedMatch = ImmutableMap.of( - "c.name.firstname", "Daenerys", - "c.parents.father", "Aerys", - "h.hname", "Targaryen", - "h.words", "fireAndBlood"); - - if (useNestedLoops) { - Assert.assertThat(hits.length(), equalTo(0)); - } else { - Assert.assertThat(hits.length(), equalTo(1)); - assertHitMatches(hits.getJSONObject(0), expectedMatch); - } - } - - private void testLeftJoin(boolean useNestedLoops) throws IOException { - - final String hint = useNestedLoops ? USE_NL_HINT : ""; - String query = String.format("SELECT%s c.name.firstname, f.name.firstname,f.name.lastname " + - "FROM %2$s c LEFT JOIN %2$s f " + - "ON f.name.firstname = c.parents.father", - hint, TEST_INDEX_GAME_OF_THRONES); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - Assert.assertThat(hits.length(), equalTo(7)); - - final Map firstMatch = new HashMap<>(); - firstMatch.put("c.name.firstname", "Daenerys"); - firstMatch.put("f.name.firstname", null); - firstMatch.put("f.name.lastname", null); - - final Map secondMatch = new HashMap<>(); - secondMatch.put("c.name.firstname", "Brandon"); - - if (useNestedLoops) { - secondMatch.put("f.name.firstname", null); - secondMatch.put("f.name.lastname", null); - } else { - secondMatch.put("f.name.firstname", "Eddard"); - secondMatch.put("f.name.lastname", "Stark"); - } - - Assert.assertTrue(hitsInclude(hits, firstMatch)); - Assert.assertTrue(hitsInclude(hits, secondMatch)); - } - - private void hintLimits_firstLimitSecondNull(boolean useNestedLoops) throws IOException { - - final String hint = useNestedLoops ? USE_NL_HINT : ""; - String query = String.format(Locale.ROOT, "SELECT%s /*! JOIN_TABLES_LIMIT(2,null) */ " + - "c.name.firstname,c.parents.father, h.hname,h.words " + - "FROM %2$s c JOIN %2$s h", - hint, TEST_INDEX_GAME_OF_THRONES); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - Assert.assertThat(hits.length(), equalTo(14)); - } - - private void hintLimits_firstLimitSecondLimit(boolean useNestedLoops) throws IOException { - - final String hint = useNestedLoops ? USE_NL_HINT : ""; - String query = String.format(Locale.ROOT, "SELECT%s /*! JOIN_TABLES_LIMIT(2,2) */ " + - "c.name.firstname,c.parents.father, h.hname,h.words FROM %2$s c " + - "JOIN %2$s h", hint, TEST_INDEX_GAME_OF_THRONES); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - Assert.assertThat(hits.length(), equalTo(4)); - } - - private void hintLimits_firstLimitSecondLimitOnlyOne(boolean useNestedLoops) throws IOException { - - final String hint = useNestedLoops ? USE_NL_HINT : ""; - String query = String.format(Locale.ROOT, "SELECT%s /*! JOIN_TABLES_LIMIT(3,1) */ " + - "c.name.firstname,c.parents.father , h.hname,h.words FROM %2$s h " + - "JOIN %2$s c ON c.name.lastname = h.hname", - hint, TEST_INDEX_GAME_OF_THRONES); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - Assert.assertThat(hits.length(), equalTo(0)); - } - - private void hintLimits_firstNullSecondLimit(boolean useNestedLoops) throws IOException { - - final String hint = useNestedLoops ? USE_NL_HINT : ""; - String query = String.format(Locale.ROOT, "SELECT%s /*! JOIN_TABLES_LIMIT(null,2) */ " + - "c.name.firstname,c.parents.father , h.hname,h.words FROM %2$s c " + - "JOIN %2$s h", hint, TEST_INDEX_GAME_OF_THRONES); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - Assert.assertThat(hits.length(), equalTo(14)); - } - - private void testLeftJoinWithLimit(boolean useNestedLoops) throws IOException { - - final String hint = useNestedLoops ? USE_NL_HINT : ""; - String query = String.format(Locale.ROOT, "SELECT%s /*! JOIN_TABLES_LIMIT(3,null) */ " + - "c.name.firstname, f.name.firstname,f.name.lastname FROM %2$s c " + - "LEFT JOIN %2$s f ON f.name.firstname = c.parents.father", - hint, TEST_INDEX_GAME_OF_THRONES); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - Assert.assertThat(hits.length(), equalTo(3)); - } - - private void joinWithOr(boolean useNestedLoops) throws IOException { - - final String hint = useNestedLoops ? USE_NL_HINT : ""; - String query = String.format(Locale.ROOT, "SELECT%s d.dog_name,c.name.firstname " + - "FROM %s c JOIN %s d " + - "ON d.holdersName = c.name.firstname OR d.age = c.name.ofHisName", - hint, TEST_INDEX_GAME_OF_THRONES, TEST_INDEX_DOG); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - - final Map firstMatch = ImmutableMap.of( - "c.name.firstname", "Daenerys", - "d.dog_name", "rex"); - final Map secondMatch = ImmutableMap.of( - "c.name.firstname", "Brandon", - "d.dog_name", "snoopy"); - - if (useNestedLoops) { - Assert.assertThat(hits.length(), equalTo(1)); - Assert.assertTrue("hits contains brandon", hitsInclude(hits, secondMatch)); - } else { - Assert.assertThat(hits.length(), equalTo(2)); - Assert.assertTrue("hits contains daenerys", hitsInclude(hits, firstMatch)); - Assert.assertTrue("hits contains brandon", hitsInclude(hits, secondMatch)); - } - } - - private void joinWithOrderFirstTable(boolean useNestedLoops) throws IOException { - - final String hint = useNestedLoops ? USE_NL_HINT : ""; - String query = String.format(Locale.ROOT, "SELECT%s c.name.firstname,d.words " + - "FROM %2$s c JOIN %2$s d ON d.hname = c.house " + - "ORDER BY c.name.firstname", - hint, TEST_INDEX_GAME_OF_THRONES); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - - if (useNestedLoops) { - Assert.assertThat(hits.length(), equalTo(0)); - } else { - - Assert.assertThat(hits.length(), equalTo(4)); - - String[] expectedNames = { "Brandon", "Daenerys", "Eddard", "Jaime" }; - - IntStream.rangeClosed(0, 3).forEach(i -> { - String firstnamePath = String.format(Locale.ROOT, "/%d/_source/c.name.firstname", i); - Assert.assertThat(hits.query(firstnamePath), equalTo(expectedNames[i])); - }); - } - } - - private boolean containsTerm(final String explainedQuery, final String termName) { - - return Pattern.compile( - Pattern.quote("\"terms\":{") - + ".*" - + Pattern.quote("\"" + termName + "\":[") - ) - .matcher(explainedQuery.replaceAll("\\s+","")) - .find(); - } - - private void joinWithNullInCondition(boolean useNestedLoops, String left, - String oper1, String oper2, int expectedNum) throws IOException { - - final String hint = useNestedLoops ? USE_NL_HINT : ""; - String query = String.format(Locale.ROOT, "SELECT%s c.name.firstname,c.parents.father,c.hname," + - "f.name.firstname,f.house,f.hname FROM %s c " + - "%s JOIN %s f ON f.name.firstname = c.parents.father " + - "%s f.house = c.hname %s f.house = c.name.firstname", - hint, TEST_INDEX_GAME_OF_THRONES, left, TEST_INDEX_GAME_OF_THRONES, oper1, oper2); - - JSONObject result = executeQuery(query); - JSONArray hits = getHits(result); - - if (useNestedLoops) { - Assert.assertThat(hits.length(), equalTo(expectedNum)); - } else { - // This branch is reserved for hash join, no test currently enters it. - // Making it fail, so that whenever tests are added for hash join, - // this method gets properly updated. - Assert.fail(); - } - } - - private boolean hitsInclude(final JSONArray actualHits, Map expectedSourceValues) { - - for (final Object hitObj : actualHits.toList()) { - - final Map hit = uncheckedGetMap(hitObj); - if (hitMatches(hit, expectedSourceValues)) { - return true; - } - } - - return false; - } - - private void assertHitMatches(final JSONObject actualHit, final Map expectedSourceValues) { - - final JSONObject src = actualHit.getJSONObject("_source"); - Assert.assertThat(src.length(), equalTo(expectedSourceValues.size())); - src.keySet().forEach(key -> { - Assert.assertTrue(expectedSourceValues.containsKey(key)); - Object value = src.get(key); - Assert.assertThat(value, equalTo(expectedSourceValues.get(key))); - }); - } - - private boolean hitMatches(final Map actualHit, final Map expectedSourceValues) { - - final Map src = uncheckedGetMap(actualHit.get("_source")); - - if (src.size() != expectedSourceValues.size()) { - return false; - } - - for (final String key: src.keySet()) { - - if (!expectedSourceValues.containsKey(key)) { - return false; - } - - Object actualValue = src.get(key); - Object expectedValue = expectedSourceValues.get(key); - if ((actualValue == null && expectedValue != null) || - (actualValue != null && expectedValue == null)) { - return false; - } else if (actualValue != null && !actualValue.equals(expectedValue)) { - return false; - } - } - - return true; - } - - @SuppressWarnings("unchecked") - private Map uncheckedGetMap(final Object mapObject) { - return (Map)mapObject; - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/MathFunctionsIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/MathFunctionsIT.java deleted file mode 100644 index 0ce6726a7d..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/MathFunctionsIT.java +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.search.SearchHit; -import org.junit.Test; - -import java.io.IOException; - -import static org.hamcrest.Matchers.closeTo; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; - -public class MathFunctionsIT extends SQLIntegTestCase { - - private static final String FROM = "FROM " + TestsConstants.TEST_INDEX_ACCOUNT; - - @Override - protected void init() throws Exception { - loadIndex(Index.ACCOUNT); - } - - @Test - public void lowerCaseFunctionCall() throws IOException { - SearchHit[] hits = query( - "SELECT abs(age - 100) AS abs" - ); - for (SearchHit hit : hits) { - double abs = (double) getField(hit, "abs"); - assertThat(abs, greaterThanOrEqualTo(0.0)); - } - } - - @Test - public void upperCaseFunctionCall() throws IOException { - SearchHit[] hits = query( - "SELECT ABS(age - 100) AS abs" - ); - for (SearchHit hit : hits) { - double abs = (double) getField(hit, "abs"); - assertThat(abs, greaterThanOrEqualTo(0.0)); - } - } - - @Test - public void eulersNumber() throws IOException { - SearchHit[] hits = query( - "SELECT E() AS e" - ); - double e = (double) getField(hits[0], "e"); - assertThat(e, equalTo(Math.E)); - } - - @Test - public void pi() throws IOException { - SearchHit[] hits = query( - "SELECT PI() AS pi" - ); - double pi = (double) getField(hits[0], "pi"); - assertThat(pi, equalTo(Math.PI)); - } - - @Test - public void expm1Function() throws IOException { - SearchHit[] hits = query( - "SELECT EXPM1(2) AS expm1" - ); - double expm1 = (double) getField(hits[0], "expm1"); - assertThat(expm1, equalTo(Math.expm1(2))); - } - - @Test - public void degreesFunction() throws IOException { - SearchHit[] hits = query( - "SELECT age, DEGREES(age) AS degrees" - ); - for (SearchHit hit : hits) { - int age = (int) getFieldFromSource(hit, "age"); - double degrees = (double) getField(hit, "degrees"); - assertThat(degrees, equalTo(Math.toDegrees(age))); - } - } - - @Test - public void radiansFunction() throws IOException { - SearchHit[] hits = query( - "SELECT age, RADIANS(age) as radians" - ); - for (SearchHit hit : hits) { - int age = (int) getFieldFromSource(hit, "age"); - double radians = (double) getField(hit, "radians"); - assertThat(radians, equalTo(Math.toRadians(age))); - } - } - - @Test - public void sin() throws IOException { - SearchHit[] hits = query( - "SELECT SIN(PI()) as sin" - ); - double sin = (double) getField(hits[0], "sin"); - assertThat(sin, equalTo(Math.sin(Math.PI))); - } - - @Test - public void asin() throws IOException { - SearchHit[] hits = query( - "SELECT ASIN(PI()) as asin" - ); - double asin = Double.valueOf((String) getField(hits[0], "asin")); - assertThat(asin, equalTo(Math.asin(Math.PI))); - } - - @Test - public void sinh() throws IOException { - SearchHit[] hits = query( - "SELECT SINH(PI()) as sinh" - ); - double sinh = (double) getField(hits[0], "sinh"); - assertThat(sinh, equalTo(Math.sinh(Math.PI))); - } - - @Test - public void power() throws IOException { - SearchHit[] hits = query( - "SELECT POWER(age, 2) AS power", - "WHERE (age IS NOT NULL) AND (balance IS NOT NULL) and (POWER(balance, 3) > 0)" - ); - double power = (double) getField(hits[0], "power"); - assertTrue(power >= 0); - } - - @Test - public void atan2() throws IOException { - SearchHit[] hits = query( - "SELECT ATAN2(age, age) AS atan2", - "WHERE (age IS NOT NULL) AND (ATAN2(age, age) > 0)" - ); - double atan2 = (double) getField(hits[0], "atan2"); - assertThat(atan2, equalTo(Math.atan2(1, 1))); - } - - @Test - public void cot() throws IOException { - SearchHit[] hits = query( - "SELECT COT(PI()) AS cot" - ); - double cot = (double) getField(hits[0], "cot"); - assertThat(cot, closeTo(1 / Math.tan(Math.PI), 0.001)); - } - - @Test - public void sign() throws IOException { - SearchHit[] hits = query( - "SELECT SIGN(E()) AS sign" - ); - double sign = (double) getField(hits[0], "sign"); - assertThat(sign, equalTo(Math.signum(Math.E))); - } - - @Test - public void logWithOneParam() throws IOException { - SearchHit[] hits = query("SELECT LOG(3) AS log"); - double log = (double) getField(hits[0], "log"); - assertThat(log, equalTo(Math.log(3))); - } - - @Test - public void logWithTwoParams() throws IOException { - SearchHit[] hits = query("SELECT LOG(2, 3) AS log"); - double log = (double) getField(hits[0], "log"); - assertThat(log, closeTo(Math.log(3)/Math.log(2), 0.0001)); - } - - @Test - public void logInAggregationShouldPass() { - assertThat( - executeQuery( - "SELECT LOG(age) FROM " + TestsConstants.TEST_INDEX_ACCOUNT - + " WHERE age IS NOT NULL GROUP BY LOG(age) ORDER BY LOG(age)", "jdbc" - ), - containsString("\"type\": \"double\"") - ); - assertThat( - executeQuery( - "SELECT LOG(2, age) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + - " WHERE age IS NOT NULL GROUP BY LOG(2, age) ORDER BY LOG(2, age)", "jdbc" - ), - containsString("\"type\": \"double\"") - ); - } - - @Test - public void log10Test() throws IOException{ - SearchHit[] hits = query("SELECT log10(1000) AS log10"); - double log10 = (double) getField(hits[0], "log10"); - assertThat(log10, equalTo(3.0)); - } - - @Test - public void ln() throws IOException { - SearchHit[] hits = query("SELECT LN(5) AS ln"); - double ln = (double) getField(hits[0], "ln"); - assertThat(ln, equalTo(Math.log(5))); - } - - @Test - public void lnInAggregationShouldPass() { - assertThat( - executeQuery( - "SELECT LN(age) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + - " WHERE age IS NOT NULL GROUP BY LN(age) ORDER BY LN(age)", "jdbc" - ), - containsString("\"type\": \"double\"") - ); - } - - @Test - public void rand() throws IOException { - SearchHit[] hits = query("SELECT RAND() AS rand", "ORDER BY rand"); - for (SearchHit hit : hits) { - double rand = (double) getField(hit, "rand"); - assertTrue(rand >= 0 && rand < 1); - } - } - - private SearchHit[] query(String select, String... statements) throws IOException { - final String response = executeQueryWithStringOutput(select + " " + FROM + " " + String.join(" ", statements)); - - final XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser( - NamedXContentRegistry.EMPTY, - LoggingDeprecationHandler.INSTANCE, - response); - return SearchResponse.fromXContent(parser).getHits().getHits(); - } - - private Object getField(SearchHit hit, String fieldName) { - return hit.field(fieldName).getValue(); - } - - private Object getFieldFromSource(SearchHit hit, String fieldName) { - return hit.getSourceAsMap().get(fieldName); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/MetaDataQueriesIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/MetaDataQueriesIT.java deleted file mode 100644 index 221e306760..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/MetaDataQueriesIT.java +++ /dev/null @@ -1,428 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import org.elasticsearch.client.Request; -import org.hamcrest.Description; -import org.hamcrest.TypeSafeMatcher; -import org.json.JSONArray; -import org.json.JSONObject; -import org.junit.Test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_GAME_OF_THRONES; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifySome; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.hasItems; -import static org.hamcrest.Matchers.not; - - -/** - * The following are tests for SHOW/DESCRIBE query support under Pretty Format Response protocol using JDBC format. - * - * Unlike SELECT queries, the JDBC format response of SHOW and DESCRIBE queries has determined "schema" fields. - * - * Since these integration tests are receiving the JSON response as output, "datarows" values can't be validated by - * key since it is a JSONArray, so the expected length of "schema" will be used instead as well as the expected - * position of the field data in "datarows". - * - * These are the outputs of "schema" for SHOW and DESCRIBE, the position of the value in "datarows" will match the - * position of the field in "schema": - * - * 1) SHOW query (based on the getTables() method listed here https://docs.oracle.com/javase/8/docs/api/java/sql/DatabaseMetaData.html) - * "schema": [ - * { - * "name": "TABLE_CAT", - * "type": "keyword" - * }, - * { - * "name": "TABLE_SCHEM", - * "type": "keyword" - * }, - * { - * "name": "TABLE_NAME", - * "type": "keyword" - * }, - * { - * "name": "TABLE_TYPE", - * "type": "keyword" - * }, - * { - * "name": "REMARKS", - * "type": "keyword" - * }, - * { - * "name": "TYPE_CAT", - * "type": "keyword" - * }, - * { - * "name": "TYPE_SCHEM", - * "type": "keyword" - * }, - * { - * "name": "TYPE_NAME", - * "type": "keyword" - * }, - * { - * "name": "SELF_REFERENCING_COL_NAME", - * "type": "keyword" - * }, - * { - * "name": "REF_GENERATION", - * "type": "keyword" - * } - * ] - * - * 2) DESCRIBE query (based on the getColumns() method listed here https://docs.oracle.com/javase/8/docs/api/java/sql/DatabaseMetaData.html) - * "schema": [ - * { - * "name": "TABLE_CAT", - * "type": "keyword" - * }, - * { - * "name": "TABLE_SCHEM", - * "type": "keyword" - * }, - * { - * "name": "TABLE_NAME", - * "type": "keyword" - * }, - * { - * "name": "COLUMN_NAME", - * "type": "keyword" - * }, - * { - * "name": "DATA_TYPE", - * "type": "integer" - * }, - * { - * "name": "TYPE_NAME", - * "type": "keyword" - * }, - * { - * "name": "COLUMN_SIZE", - * "type": "integer" - * }, - * { - * "name": "BUFFER_LENGTH", - * "type": "integer" - * }, - * { - * "name": "DECIMAL_DIGITS", - * "type": "integer" - * }, - * { - * "name": "NUM_PREC_RADIX", - * "type": "integer" - * }, - * { - * "name": "NULLABLE", - * "type": "integer" - * }, - * { - * "name": "REMARKS", - * "type": "keyword" - * }, - * { - * "name": "COLUMN_DEF", - * "type": "keyword" - * }, - * { - * "name": "SQL_DATA_TYPE", - * "type": "integer" - * }, - * { - * "name": "SQL_DATETIME_SUB", - * "type": "integer" - * }, - * { - * "name": "CHAR_OCTET_LENGTH", - * "type": "integer" - * }, - * { - * "name": "ORDINAL_POSITION", - * "type": "integer" - * }, - * { - * "name": "IS_NULLABLE", - * "type": "keyword" - * }, - * { - * "name": "SCOPE_CATALOG", - * "type": "keyword" - * }, - * { - * "name": "SCOPE_SCHEMA", - * "type": "keyword" - * }, - * { - * "name": "SCOPE_TABLE", - * "type": "keyword" - * }, - * { - * "name": "SOURCE_DATA_TYPE", - * "type": "short" - * }, - * { - * "name": "IS_AUTOINCREMENT", - * "type": "keyword" - * }, - * { - * "name": "IS_GENERATEDCOLUMN", - * "type": "keyword" - * } - * ] - */ -public class MetaDataQueriesIT extends SQLIntegTestCase { - - // Number of fields in the response, assuming no filters, based on output shown above - private static final int SHOW_FIELD_LENGTH = 10; - - private static final int DESCRIBE_FIELD_LENGTH = 24; - - private static final String TABLE_TYPE = "BASE TABLE"; - - @Override - protected void init() throws Exception { - loadIndex(Index.ACCOUNT); - loadIndex(Index.PHRASE); - loadIndex(Index.GAME_OF_THRONES); - } - - @Override - protected Request getSqlRequest(String request, boolean explain) { - Request sqlRequest = super.getSqlRequest(request, explain); - sqlRequest.addParameter("format", "jdbc"); - - return sqlRequest; - } - - @Test - public void showSingleIndex() throws IOException { - JSONObject response = executeQuery(String.format("SHOW TABLES LIKE %s", TestsConstants.TEST_INDEX_ACCOUNT)); - - String[] fields = {"TABLE_CAT", "TABLE_NAME", "TABLE_TYPE"}; - checkContainsColumns(getSchema(response), fields); - - JSONArray dataRows = getDataRows(response); - assertThat(dataRows.length(), equalTo(1)); - assertThat(dataRows.getJSONArray(0).length(), equalTo(SHOW_FIELD_LENGTH)); - - /* - * Assumed indices of fields in dataRows based on "schema" output for SHOW given above: - * "TABLE_CAT" : 0 - * "TABLE_NAME" : 2 - * "TABLE_TYPE" : 3 - */ - JSONArray row = dataRows.getJSONArray(0); - assertThat(row.get(0), equalTo(getClusterName())); - assertThat(row.get(2), equalTo(TestsConstants.TEST_INDEX_ACCOUNT)); - assertThat(row.get(3), equalTo(TABLE_TYPE)); - } - - @Test - public void showCaseSensitivityCheck() throws IOException { - JSONObject response = executeQuery(String.format("show tables like %s", TestsConstants.TEST_INDEX_ACCOUNT)); - - String[] fields = {"TABLE_CAT", "TABLE_NAME", "TABLE_TYPE"}; - checkContainsColumns(getSchema(response), fields); - - JSONArray dataRows = getDataRows(response); - assertThat(dataRows.length(), equalTo(1)); - assertThat(dataRows.getJSONArray(0).length(), equalTo(SHOW_FIELD_LENGTH)); - - JSONArray row = dataRows.getJSONArray(0); - assertThat(row.get(0), equalTo(getClusterName())); - assertThat(row.get(2), equalTo(TestsConstants.TEST_INDEX_ACCOUNT)); - assertThat(row.get(3), equalTo(TABLE_TYPE)); - } - - @Test - public void showWildcardIndex() throws IOException { - JSONObject response = executeQuery(String.format("SHOW TABLES LIKE %s%%", TestsConstants.TEST_INDEX)); - - String pattern = String.format("%s.*", TestsConstants.TEST_INDEX); - JSONArray dataRows = getDataRows(response); - assertThat(dataRows.length(), equalTo(3)); - for (int i = 0; i < dataRows.length(); i++) { - JSONArray row = dataRows.getJSONArray(i); - String tableName = row.getString(2); - - assertTrue(tableName.matches(pattern)); - } - } - - @Test - public void describeSingleIndex() throws IOException { - JSONObject response = executeQuery(String.format("DESCRIBE TABLES LIKE %s", TestsConstants.TEST_INDEX_ACCOUNT)); - - // Schema for DESCRIBE is filled with a lot of fields that aren't used so only the important - // ones are checked for here - String[] fields = {"TABLE_NAME", "COLUMN_NAME", "TYPE_NAME"}; - checkContainsColumns(getSchema(response), fields); - - JSONArray dataRows = getDataRows(response); - assertThat(dataRows.length(), greaterThan(0)); - assertThat(dataRows.getJSONArray(0).length(), equalTo(DESCRIBE_FIELD_LENGTH)); - - /* - * Assumed indices of fields in dataRows based on "schema" output for DESCRIBE given above: - * "TABLE_NAME" : 2 - * "COLUMN_NAME" : 3 - * "TYPE_NAME" : 5 - */ - JSONArray row = dataRows.getJSONArray(0); - assertThat(row.get(2), equalTo(TestsConstants.TEST_INDEX_ACCOUNT)); - assertThat(row.get(3), not(equalTo(JSONObject.NULL))); - assertThat(row.get(5), not(equalTo(JSONObject.NULL))); - } - - @Test - public void describeSingleIndexWithObjectFieldShouldPass() throws IOException { - JSONObject response = - executeQuery(String.format("DESCRIBE TABLES LIKE %s", TEST_INDEX_GAME_OF_THRONES)); - - // Schema for DESCRIBE is filled with a lot of fields that aren't used so only the important - // ones are checked for here - String[] fields = {"TABLE_NAME", "COLUMN_NAME", "TYPE_NAME"}; - checkContainsColumns(getSchema(response), fields); - - JSONArray dataRows = getDataRows(response); - assertThat(dataRows.length(), greaterThan(0)); - assertThat(dataRows.getJSONArray(0).length(), equalTo(DESCRIBE_FIELD_LENGTH)); - - verifySome(dataRows, - describeRow(TEST_INDEX_GAME_OF_THRONES, "nickname", "text"), - describeRow(TEST_INDEX_GAME_OF_THRONES, "name", "object"), - describeRow(TEST_INDEX_GAME_OF_THRONES, "name.firstname", "text"), - describeRow(TEST_INDEX_GAME_OF_THRONES, "name.lastname", "text"), - describeRow(TEST_INDEX_GAME_OF_THRONES, "name.ofHerName", "integer"), - describeRow(TEST_INDEX_GAME_OF_THRONES, "name.ofHisName", "integer"), - describeRow(TEST_INDEX_GAME_OF_THRONES, "house", "text"), - describeRow(TEST_INDEX_GAME_OF_THRONES, "gender", "text")); - } - - @Test - public void describeCaseSensitivityCheck() throws IOException { - JSONObject response = executeQuery(String.format("describe tables like %s", TestsConstants.TEST_INDEX_ACCOUNT)); - - String[] fields = {"TABLE_NAME", "COLUMN_NAME", "TYPE_NAME"}; - checkContainsColumns(getSchema(response), fields); - - JSONArray dataRows = getDataRows(response); - assertThat(dataRows.length(), greaterThan(0)); - assertThat(dataRows.getJSONArray(0).length(), equalTo(DESCRIBE_FIELD_LENGTH)); - - JSONArray row = dataRows.getJSONArray(0); - assertThat(row.get(2), equalTo(TestsConstants.TEST_INDEX_ACCOUNT)); - assertThat(row.get(3), not(equalTo(JSONObject.NULL))); - assertThat(row.get(5), not(equalTo(JSONObject.NULL))); - } - - @Test - public void describeWildcardIndex() throws IOException { - JSONObject response = executeQuery(String.format("DESCRIBE TABLES LIKE %s%%", TestsConstants.TEST_INDEX)); - - String pattern = String.format("%s.*", TestsConstants.TEST_INDEX); - JSONArray dataRows = getDataRows(response); - assertThat(dataRows.length(), greaterThan(0)); - for (int i = 0; i < dataRows.length(); i++) { - JSONArray row = dataRows.getJSONArray(i); - String tableName = row.getString(2); - - assertTrue(tableName.matches(pattern)); - } - } - - @Test - public void describeWildcardColumn() throws IOException { - JSONObject response = executeQuery(String.format("DESCRIBE TABLES LIKE %s COLUMNS LIKE %%name", - TestsConstants.TEST_INDEX_ACCOUNT)); - - String pattern = ".*name"; - JSONArray dataRows = getDataRows(response); - assertThat(dataRows.length(), greaterThan(0)); - for (int i = 0; i < dataRows.length(); i++) { - JSONArray row = dataRows.getJSONArray(i); - String columnName = row.getString(3); - - assertTrue(columnName.matches(pattern)); - } - } - - @Test - public void describeSingleCharacterWildcard() throws IOException { - JSONObject response = executeQuery(String.format("DESCRIBE TABLES LIKE %s COLUMNS LIKE %%na_e", - TestsConstants.TEST_INDEX_ACCOUNT)); - - String pattern = ".*na.e"; - JSONArray dataRows = getDataRows(response); - assertThat(dataRows.length(), greaterThan(0)); - for (int i = 0; i < dataRows.length(); i++) { - JSONArray row = dataRows.getJSONArray(i); - String columnName = row.getString(3); - - assertTrue(columnName.matches(pattern)); - } - } - - private JSONArray getSchema(JSONObject jdbcResponse) { return jdbcResponse.getJSONArray("schema"); } - - private JSONArray getDataRows(JSONObject jdbcResponse) { - return jdbcResponse.getJSONArray("datarows"); - } - - private void checkContainsColumns(JSONArray schema, String... fields) { - List columnNames = new ArrayList<>(); - for (int i = 0; i < schema.length(); i++) { - JSONObject column = schema.getJSONObject(i); - columnNames.add(column.getString("name")); - } - - assertThat(columnNames, hasItems(fields)); - } - - private String getClusterName() throws IOException { - String response = executeRequest(new Request("GET", "_cluster/health")); - return new JSONObject(response).optString("cluster_name", ""); - } - - public static TypeSafeMatcher describeRow(Object... expectedObjects) { - return new TypeSafeMatcher() { - @Override - public void describeTo(Description description) { - description.appendText(String.join(",", Arrays.asList(expectedObjects).toString())); - } - - @Override - protected boolean matchesSafely(JSONArray array) { - List actualObjects = new ArrayList<>(); - // TABLE_NAME - actualObjects.add(array.get(2)); - // COLUMN_NAME - actualObjects.add(array.get(3)); - // TYPE_NAME - actualObjects.add(array.get(5)); - return Arrays.asList(expectedObjects).equals(actualObjects); - } - }; - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/MethodQueryIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/MethodQueryIT.java deleted file mode 100644 index ff1f6aa59e..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/MethodQueryIT.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import org.junit.Assert; -import org.junit.Test; - -import java.io.IOException; -import java.util.Locale; - -import static org.hamcrest.Matchers.both; -import static org.hamcrest.Matchers.containsString; - -/** - * 定製方法查詢． - * - * @author ansj - */ -public class MethodQueryIT extends SQLIntegTestCase { - - @Override - protected void init() throws Exception { - loadIndex(Index.ACCOUNT); - } - - /** - * query 搜索就是　，　lucene 原生的搜素方式 注意这个例子中ｖａｌｕｅ可以随便命名 "query" : - * {query_string" : {"query" : "address:880 Holmes Lane"} - * - * @throws IOException - */ - @Test - public void queryTest() throws IOException { - final String result = explainQuery(String.format(Locale.ROOT, - "select address from %s where query('address:880 Holmes Lane') limit 3", - TestsConstants.TEST_INDEX_ACCOUNT)); - Assert.assertThat(result, - containsString("query_string\":{\"query\":\"address:880 Holmes Lane")); - - } - - /** - * matchQuery 是利用分词结果进行单个字段的搜索． "query" : { "match" : { "address" : - * {"query":"880 Holmes Lane", "type" : "boolean" } } } - * - * @throws IOException - */ - @Test - public void matchQueryTest() throws IOException { - final String result = explainQuery(String.format(Locale.ROOT, - "select address from %s where address= matchQuery('880 Holmes Lane') limit 3", - TestsConstants.TEST_INDEX_ACCOUNT)); - Assert.assertThat(result, - containsString("{\"match\":{\"address\":{\"query\":\"880 Holmes Lane\"")); - } - - /** - * matchQuery 是利用分词结果进行单个字段的搜索． "query" : { "bool" : { "must" : { "bool" : { - * "should" : [ { "constant_score" : { "query" : { "match" : { "address" : { - * "query" : "Lane", "type" : "boolean" } } }, "boost" : 100.0 } }, { - * "constant_score" : { "query" : { "match" : { "address" : { "query" : - * "Street", "type" : "boolean" } } }, "boost" : 0.5 } } ] } } } } - * - * @throws IOException - */ - // todo - @Test - public void scoreQueryTest() throws IOException { - final String result = explainQuery(String.format(Locale.ROOT, - "select address from %s " + - "where score(matchQuery(address, 'Lane'),100) " + - "or score(matchQuery(address,'Street'),0.5) order by _score desc limit 3", - TestsConstants.TEST_INDEX_ACCOUNT)); - Assert.assertThat(result, - both(containsString("{\"constant_score\":" + - "{\"filter\":{\"match\":{\"address\":{\"query\":\"Lane\"")).and( - containsString("{\"constant_score\":" + - "{\"filter\":{\"match\":{\"address\":{\"query\":\"Street\""))); - } - - /** - * wildcardQuery 是用通配符的方式查找某个term 　比如例子中 l*e means leae ltae .... - * "wildcard": { "address" : { "wildcard" : "l*e" } } - * - * @throws IOException - */ - @Test - public void wildcardQueryTest() throws IOException { - final String result = explainQuery(String.format(Locale.ROOT, - "select address from %s where address= wildcardQuery('l*e') order by _score desc limit 3", - TestsConstants.TEST_INDEX_ACCOUNT)); - Assert.assertThat(result, - containsString("{\"wildcard\":{\"address\":{\"wildcard\":\"l*e\"")); - } - - /** - * matchPhraseQueryTest 短语查询完全匹配． - * "address" : { - * "query" : "671 Bristol Street", - * "type" : "phrase" - * } - * - * @throws IOException - */ - @Test - public void matchPhraseQueryTest() throws IOException { - final String result = explainQuery(String.format(Locale.ROOT, - "select address from %s " + - "where address= matchPhrase('671 Bristol Street') order by _score desc limit 3", - TestsConstants.TEST_INDEX_ACCOUNT)); - Assert.assertThat(result, - containsString("{\"match_phrase\":{\"address\":{\"query\":\"671 Bristol Street\"")); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/MetricsIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/MetricsIT.java deleted file mode 100644 index e1dbbaef16..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/MetricsIT.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import com.amazon.opendistroforelasticsearch.sql.metrics.MetricName; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.json.JSONObject; -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.concurrent.TimeUnit; - -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_DOG; -import static org.hamcrest.Matchers.equalTo; - -@Ignore -public class MetricsIT extends SQLIntegTestCase { - - @Override - protected void init() throws Exception { - loadIndex(Index.ACCOUNT); - loadIndex(Index.DOG); - } - - @Test - public void requestCount() throws IOException, InterruptedException { - multiQueries(3); - TimeUnit.SECONDS.sleep(2L); - JSONObject jsonObject = new JSONObject(executeStatRequest(makeStatRequest())); - assertThat(jsonObject.getInt(MetricName.REQ_COUNT_TOTAL.getName()), equalTo(3)); - } - - private void multiQueries(int n) throws IOException { - for (int i=0; i names = new HashSet<>(); - JSONArray hits = getHits(response); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - JSONObject source = getSource(hit); - - names.add(source.getString("firstname")); - } - - assertThat(names, hasItems("Amber", "rex")); - } - - @Test - public void unionAllOnlyOneRecordEachWithComplexAlias() throws IOException { - String query = String.format("SELECT firstname FROM %s WHERE firstname = 'Amber' " + - "UNION ALL " + - "SELECT name.firstname as firstname " + - "FROM %s " + - "WHERE name.firstname = 'daenerys'", - TestsConstants.TEST_INDEX_ACCOUNT, TestsConstants.TEST_INDEX_GAME_OF_THRONES); - - JSONObject response = executeQuery(query); - assertThat(getHits(response).length(), equalTo(2)); - - Set names = new HashSet<>(); - JSONArray hits = getHits(response); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - JSONObject source = getSource(hit); - - names.add(source.getString("firstname")); - } - - assertThat(names, hasItems("Amber", "Daenerys")); - } - - @Test - public void minusAMinusANoAlias() throws IOException { - innerMinusAMinusANoAlias(""); - } - - @Test - public void minusAMinusANoAliasWithScrolling() throws IOException { - innerMinusAMinusANoAlias(MINUS_SCROLL_DEFAULT_HINT); - } - - @Test - public void minusAMinusANoAliasWithScrollingAndTerms() throws IOException { - innerMinusAMinusANoAlias(MINUS_SCROLL_DEFAULT_HINT + MINUS_TERMS_OPTIMIZATION_HINT); - } - - @Test - public void minusAMinusBNoAlias() throws IOException { - innerMinusAMinusBNoAlias(""); - } - - @Test - public void minusAMinusBNoAliasWithScrolling() throws IOException { - innerMinusAMinusBNoAlias(MINUS_SCROLL_DEFAULT_HINT); - } - - @Test - public void minusAMinusBNoAliasWithScrollingAndTerms() throws IOException { - innerMinusAMinusBNoAlias(MINUS_SCROLL_DEFAULT_HINT + MINUS_TERMS_OPTIMIZATION_HINT); - } - - @Test - public void minusCMinusDTwoFieldsNoAlias() throws IOException { - innerMinusCMinusDTwoFieldsNoAlias(""); - } - - @Test - public void minusCMinusDTwoFieldsNoAliasWithScrolling() throws IOException { - innerMinusCMinusDTwoFieldsNoAlias(MINUS_SCROLL_DEFAULT_HINT); - } - - @Test - public void minusCMinusDTwoFieldsAliasOnBothSecondTableFields() throws IOException { - String query = String.format("SELECT pk, letter FROM %s WHERE system_name = 'C' " + - "MINUS " + - "SELECT myId as pk, myLetter as letter FROM %s WHERE system_name = 'E'", - TestsConstants.TEST_INDEX_SYSTEM, TestsConstants.TEST_INDEX_SYSTEM); - - JSONObject response = executeQuery(query); - assertThat(getHits(response).length(), equalTo(1)); - - JSONObject hit = getHits(response).getJSONObject(0); - JSONObject source = getSource(hit); - - assertThat(source.length(), equalTo(2)); - Assert.assertTrue("Source expected to contain 'pk' field", source.has("pk")); - Assert.assertTrue("Source expected to contain 'letter' field", source.has("letter")); - assertThat(source.getInt("pk"), equalTo(1)); - assertThat(source.getString("letter"), equalTo("e")); - } - - @Test - public void minusCMinusDTwoFieldsAliasOnBothTables() throws IOException { - innerMinusCMinusDTwoFieldsAliasOnBothTables(""); - } - - @Test - public void minusCMinusDTwoFieldsAliasOnBothTablesWithScrolling() throws IOException { - innerMinusCMinusDTwoFieldsAliasOnBothTables(MINUS_SCROLL_DEFAULT_HINT); - } - - @Test - public void minusCMinusCTwoFieldsOneAlias() throws IOException { - String query = String.format("SELECT pk as myId, letter FROM %s WHERE system_name = 'C' " + - "MINUS " + - "SELECT pk as myId, letter FROM %s WHERE system_name = 'C'", - TestsConstants.TEST_INDEX_SYSTEM, TestsConstants.TEST_INDEX_SYSTEM); - - JSONObject response = executeQuery(query); - assertThat(getHits(response).length(), equalTo(0)); - } - - @Test - public void minusCMinusTNonExistentFieldTwoFields() throws IOException { - String query = String.format("SELECT pk, letter FROM %s WHERE system_name = 'C' " + - "MINUS " + - "SELECT pk, letter FROM %s WHERE system_name = 'T' ", - TestsConstants.TEST_INDEX_SYSTEM, TestsConstants.TEST_INDEX_SYSTEM); - - JSONObject response = executeQuery(query); - assertThat(getHits(response).length(), equalTo(3)); - } - - @Test - public void minusCMinusTNonExistentFieldOneField() throws IOException { - innerMinusCMinusTNonExistentFieldOneField(""); - } - - @Test - public void minusCMinusTNonExistentOneFieldWithScrolling() throws IOException { - innerMinusCMinusTNonExistentFieldOneField(MINUS_SCROLL_DEFAULT_HINT); - } - - @Test - public void minusCMinusTNonExistentFieldOneFieldWithScrollingAndOptimization() throws IOException { - innerMinusCMinusTNonExistentFieldOneField(MINUS_SCROLL_DEFAULT_HINT + MINUS_TERMS_OPTIMIZATION_HINT); - } - - @Test - public void minusTMinusCNonExistentFieldFirstQuery() throws IOException { - innerMinusTMinusCNonExistentFieldFirstQuery(""); - } - - @Test - public void minusTMinusCNonExistentFieldFirstQueryWithScrolling() throws IOException { - innerMinusTMinusCNonExistentFieldFirstQuery(MINUS_SCROLL_DEFAULT_HINT); - } - - @Test - public void minusTMinusCNonExistentFieldFirstQueryWithScrollingAndOptimization() throws IOException { - innerMinusTMinusCNonExistentFieldFirstQuery(MINUS_SCROLL_DEFAULT_HINT + MINUS_TERMS_OPTIMIZATION_HINT); - } - - private void innerMinusAMinusANoAlias(String hint) throws IOException { - String query = String.format("SELECT %s pk FROM %s WHERE system_name = 'A' " + - "MINUS " + - "SELECT pk FROM %s WHERE system_name = 'A'", - hint, TestsConstants.TEST_INDEX_SYSTEM, TestsConstants.TEST_INDEX_SYSTEM); - - JSONObject response = executeQuery(query); - assertThat(getHits(response).length(), equalTo(0)); - } - - private void innerMinusAMinusBNoAlias(String hint) throws IOException { - String query = String.format("SELECT %s pk FROM %s WHERE system_name = 'A' " + - "MINUS " + - "SELECT pk FROM %s WHERE system_name = 'B'", - hint, TestsConstants.TEST_INDEX_SYSTEM, TestsConstants.TEST_INDEX_SYSTEM); - - JSONObject response = executeQuery(query); - assertThat(getHits(response).length(), equalTo(1)); - - JSONObject hit = getHits(response).getJSONObject(0); - JSONObject source = getSource(hit); - assertThat(source.length(), equalTo(1)); - Assert.assertTrue("Source expected to contain 'pk' field", source.has("pk")); - assertThat(source.getInt("pk"), equalTo(3)); - } - - private void innerMinusCMinusDTwoFieldsNoAlias(String hint) throws IOException { - String query = String.format("SELECT %s pk, letter FROM %s WHERE system_name = 'C' " + - "MINUS " + - "SELECT pk, letter FROM %s WHERE system_name = 'D'", - hint, TestsConstants.TEST_INDEX_SYSTEM, TestsConstants.TEST_INDEX_SYSTEM); - - JSONObject response = executeQuery(query); - assertThat(getHits(response).length(), equalTo(1)); - - JSONObject hit = getHits(response).getJSONObject(0); - JSONObject source = getSource(hit); - - assertThat(source.length(), equalTo(2)); - Assert.assertTrue("Source expected to contain 'pk' field", source.has("pk")); - Assert.assertTrue("Source expected to contain 'letter' field", source.has("letter")); - assertThat(source.getInt("pk"), equalTo(1)); - assertThat(source.getString("letter"), equalTo("e")); - } - - private void innerMinusCMinusDTwoFieldsAliasOnBothTables(String hint) throws IOException { - String query = String.format("SELECT %s pk as myId, letter FROM %s WHERE system_name = 'C' " + - "MINUS " + - "SELECT myId, myLetter as letter FROM %s WHERE system_name = 'E'", - hint, TestsConstants.TEST_INDEX_SYSTEM, TestsConstants.TEST_INDEX_SYSTEM); - - JSONObject response = executeQuery(query); - assertThat(getHits(response).length(), equalTo(1)); - - JSONObject hit = getHits(response).getJSONObject(0); - JSONObject source = getSource(hit); - - assertThat(source.length(), equalTo(2)); - Assert.assertTrue("Source expected to contain 'pk' field as 'myId'", source.has("myId")); - Assert.assertTrue("Source expected to contain 'letter' field", source.has("letter")); - assertThat(source.getInt("myId"), equalTo(1)); - assertThat(source.getString("letter"), equalTo("e")); - } - - private void innerMinusCMinusTNonExistentFieldOneField(String hint) throws IOException { - String query = String.format("SELECT %s letter FROM %s WHERE system_name = 'C' " + - "MINUS " + - "SELECT letter FROM %s WHERE system_name = 'T'", - hint, TestsConstants.TEST_INDEX_SYSTEM, TestsConstants.TEST_INDEX_SYSTEM); - - JSONObject response = executeQuery(query); - assertThat(getHits(response).length(), equalTo(3)); - } - - private void innerMinusTMinusCNonExistentFieldFirstQuery(String hint) throws IOException { - String query = String.format("SELECT %s letter FROM %s WHERE system_name = 'T' " + - "MINUS " + - "SELECT letter FROM %s WHERE system_name = 'C'", - hint, TestsConstants.TEST_INDEX_SYSTEM, TestsConstants.TEST_INDEX_SYSTEM); - - JSONObject response = executeQuery(query); - assertThat(getHits(response).length(), equalTo(0)); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/NestedFieldQueryIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/NestedFieldQueryIT.java deleted file mode 100644 index e6ea967b9c..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/NestedFieldQueryIT.java +++ /dev/null @@ -1,908 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.ResponseException; -import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.search.SearchHit; -import org.hamcrest.BaseMatcher; -import org.hamcrest.Description; -import org.hamcrest.FeatureMatcher; -import org.hamcrest.Matcher; -import org.hamcrest.core.Is; -import org.json.JSONArray; -import org.json.JSONObject; -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.function.Function; - -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.hitAll; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.kvString; -import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.arrayContainingInAnyOrder; -import static org.hamcrest.Matchers.closeTo; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; - -/** - * Integration test cases for both rewriting and projection logic. - * - * Test result: - * 1) SELECT * or any field or aggregate function on any field - * 2) WHERE single or multiple conditions on nested type - * 3) GROUP BY regular field and aggregate on any field - * 4) GROUP BY nested field and COUNT(*) - * 5) UNION/MINUS but only SELECT nested field - * - * Does NOT support: - * 1) GROUP BY nested field and aggregate other than COUNT - * 2) UNION/MINUS and SELECT nested field (need to flatten during set computation) - * 3) JOIN (doesn't work if put alias before nested field name and may have similar problem as UNION/MINUS during computation) - * 4) Subquery - * 5) HAVING - * 6) Verification for conditions mixed with regular and nested fields - */ -public class NestedFieldQueryIT extends SQLIntegTestCase { - - private static final String FROM = "FROM " + TestsConstants.TEST_INDEX_NESTED_TYPE + " n, n.message m"; - - - @Override - protected void init() throws Exception { - loadIndex(Index.NESTED); - loadIndex(Index.EMPLOYEE_NESTED); - } - - @Test - public void selectAll() throws IOException { - queryAll("SELECT *"); - } - - @Test - public void noCondition() throws IOException { - queryAll("SELECT myNum, someField, m.author, m.info"); - } - - private void queryAll(String sql) throws IOException { - assertThat( - query(sql), - hits( - hit( - myNum(1), - someField("b"), - innerHits("message", - hit( - author("e"), - info("a") - ) - ) - ), - hit( - myNum(2), - someField("a"), - innerHits("message", - hit( - author("f"), - info("b") - ) - ) - ), - hit( - myNum(3), - someField("a"), - innerHits("message", - hit( - author("g"), - info("c") - ) - ) - ), - hit( - myNum(4), - someField("b"), - innerHits("message", - hit( - author("h"), - info("c") - ), - hit( - author("i"), - info("a") - ) - ) - ), - hit( - myNum(new int[]{3, 4}), - someField("a"), - innerHits("message", - hit( - author("zz"), - info("zz") - ) - ) - ) - ) - ); - } - - @Test - public void singleCondition() throws IOException { - assertThat( - query( - "SELECT myNum, m.author, m.info", - "WHERE m.info = 'c'" - ), - hits( - hit( - myNum(3), - innerHits("message", - hit( - author("g"), - info("c") - ) - ) - ), - hit( - myNum(4), - innerHits("message", - hit( - author("h"), - info("c") - ) - ) - ) - ) - ); - } - - @Test - public void multipleConditionsOfNestedField() throws IOException { - assertThat( - query( - "SELECT someField, m.author, m.info", - "WHERE m.info = 'c' AND m.author = 'h'" - ), - hits( - hit( - someField("b"), - innerHits("message", - hit( - author("h"), - info("c") - ) - ) - ) - ) - ); - } - - @Test - public void multipleConditionsOfNestedFieldNoMatch() throws IOException { - assertThat( - query( - "SELECT someField, m.author, m.info", - "WHERE m.info = 'c' AND m.author = 'i'" - ), - hits() - ); - } - - @Test - public void multipleConditionsOfRegularAndNestedField() throws IOException { - assertThat( - query( - "SELECT myNum, m.author, m.info", - "WHERE myNum = 3 AND m.info = 'c'" - ), - hits( - hit( - myNum(3), - innerHits("message", - hit( - author("g"), - info("c") - ) - ) - ) - ) - ); - } - - @Test - public void multipleConditionsOfRegularOrNestedField() throws IOException { - assertThat( - query( - "SELECT myNum, m.author, m.info", - "WHERE myNum = 2 OR m.info = 'c'" - ), - hits( - hit( - myNum(2) - ), // Note: no inner hit here because of no match in nested field - hit( - myNum(3), - innerHits("message", - hit( - author("g"), - info("c") - ) - ) - ), - hit( - myNum(4), - innerHits("message", - hit( - author("h"), - info("c") - ) - ) - ) - ) - ); - } - - @Test - public void leftJoinSelectAll() throws IOException { - String sql = "SELECT * " + - "FROM elasticsearch-sql_test_index_employee_nested e " + - "LEFT JOIN e.projects p"; - String explain = explainQuery(sql); - assertThat(explain, containsString("{\"bool\":{\"must_not\":[{\"nested\":{\"query\":" + - "{\"exists\":{\"field\":\"projects\",\"boost\":1.0}},\"path\":\"projects\"")); - - assertThat(explain, containsString("\"_source\":{\"includes\":[\"projects.*\"")); - - JSONObject results = executeQuery(sql); - Assert.assertThat(getTotalHits(results), equalTo(4)); - } - - @Test - public void leftJoinSpecificFields() throws IOException { - String sql = "SELECT e.name, p.name, p.started_year " + - "FROM elasticsearch-sql_test_index_employee_nested e " + - "LEFT JOIN e.projects p"; - String explain = explainQuery(sql); - assertThat(explain, containsString("{\"bool\":{\"must_not\":[{\"nested\":{\"query\":" + - "{\"exists\":{\"field\":\"projects\",\"boost\":1.0}},\"path\":\"projects\"")); - assertThat(explain, containsString("\"_source\":{\"includes\":[\"name\"],")); - assertThat(explain, containsString("\"_source\":{\"includes\":[\"projects.name\",\"projects.started_year\"]")); - - JSONObject results = executeQuery(sql); - Assert.assertThat(getTotalHits(results), equalTo(4)); - } - - @Ignore("Comma join in left join won't pass syntax check in new ANTLR parser. " - + "Ignore for now and require to change grammar too when we want to support this case.") - @Test - public void leftJoinExceptionOnExtraNestedFields() throws IOException { - String sql = "SELECT * " + - "FROM elasticsearch-sql_test_index_employee_nested e " + - "LEFT JOIN e.projects p, e.comments c"; - - try { - String explain = explainQuery(sql); - Assert.fail("Expected ResponseException, but none was thrown"); - } catch (ResponseException e) { - assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.BAD_REQUEST.getStatus())); - final String entity = TestUtils.getResponseBody(e.getResponse()); - assertThat(entity, containsString("only single nested field is allowed as right table for LEFT JOIN")); - assertThat(entity, containsString("\"type\":\"verification_exception\"")); - } - } - - - @Test - public void aggregationWithoutGroupBy() throws IOException { - String sql = "SELECT AVG(m.dayOfWeek) AS avgDay " + FROM; - - JSONObject result = executeQuery(sql); - JSONObject aggregation = getAggregation(result, "message.dayOfWeek@NESTED"); - - Assert.assertThat((Double) aggregation.query("/avgDay/value"), closeTo(3.166666666, 0.01)); - } - - @Test - public void groupByNestedFieldAndCount() throws IOException { - final String sql = "SELECT m.info, COUNT(*) " + FROM + " GROUP BY m.info"; - - JSONObject result = executeQuery(sql); - JSONObject aggregation = getAggregation(result, "message.info@NESTED"); - JSONArray msgInfoBuckets = (JSONArray)aggregation.optQuery("/message.info/buckets"); - - Assert.assertNotNull(msgInfoBuckets); - Assert.assertThat(msgInfoBuckets.length(), equalTo(4)); - Assert.assertThat(msgInfoBuckets.query("/0/key"), equalTo("a")); - Assert.assertThat(msgInfoBuckets.query("/0/COUNT(*)/value"), equalTo(2)); - Assert.assertThat(msgInfoBuckets.query("/1/key"), equalTo("c")); - Assert.assertThat(msgInfoBuckets.query("/1/COUNT(*)/value"), equalTo(2)); - Assert.assertThat(msgInfoBuckets.query("/2/key"), equalTo("b")); - Assert.assertThat(msgInfoBuckets.query("/2/COUNT(*)/value"), equalTo(1)); - Assert.assertThat(msgInfoBuckets.query("/3/key"), equalTo("zz")); - Assert.assertThat(msgInfoBuckets.query("/3/COUNT(*)/value"), equalTo(1)); - } - - @Test - public void groupByRegularFieldAndSum() throws IOException { - final String sql = "SELECT *, SUM(m.dayOfWeek) AS sumDay " + FROM + " GROUP BY someField"; - - JSONObject result = executeQuery(sql); - JSONObject aggregation = getAggregation(result, "someField"); - JSONArray msgInfoBuckets = (JSONArray)aggregation.optQuery("/buckets"); - - Assert.assertNotNull(msgInfoBuckets); - Assert.assertThat(msgInfoBuckets.length(), equalTo(2)); - Assert.assertThat(msgInfoBuckets.query("/0/key"), equalTo("a")); - Assert.assertThat((Double) msgInfoBuckets.query("/0/message.dayOfWeek@NESTED/sumDay/value"), closeTo(9.0, 0.01)); - Assert.assertThat(msgInfoBuckets.query("/1/key"), equalTo("b")); - Assert.assertThat((Double) msgInfoBuckets.query("/1/message.dayOfWeek@NESTED/sumDay/value"), closeTo(10.0, 0.01)); - } - - @Test - public void nestedFiledIsNotNull() throws IOException { - String sql = "SELECT e.name " + - "FROM elasticsearch-sql_test_index_employee_nested as e, e.projects as p " + - "WHERE p IS NOT NULL"; - - assertThat( - executeQuery(sql), - hitAll( - kvString("/_source/name", Is.is("Bob Smith")), - kvString("/_source/name", Is.is("Jane Smith")) - ) - ); - } - - // Doesn't support: aggregate function other than COUNT() - @SuppressWarnings("unused") - public void groupByNestedFieldAndAvg() throws IOException { - query( - "SELECT m.info, AVG(m.dayOfWeek)", - "GROUP BY m.info" - ); - query( - "SELECT m.info, AVG(myNum)", - "GROUP BY m.info" - ); - } - - @Test - public void groupByNestedAndRegularField() throws IOException { - final String sql = "SELECT someField, m.info, COUNT(*) " + FROM + " GROUP BY someField, m.info"; - - JSONObject result = executeQuery(sql); - JSONObject aggregation = getAggregation(result, "someField"); - JSONArray msgInfoBuckets = (JSONArray)aggregation.optQuery("/buckets"); - - Assert.assertNotNull(msgInfoBuckets); - Assert.assertThat(msgInfoBuckets.length(), equalTo(2)); - Assert.assertThat(msgInfoBuckets.query("/0/key"), equalTo("a")); - Assert.assertThat(msgInfoBuckets.query("/1/key"), equalTo("b")); - - JSONArray innerBuckets = (JSONArray)msgInfoBuckets.optQuery("/0/message.info@NESTED/message.info/buckets"); - Assert.assertThat(innerBuckets.query("/0/key"), equalTo("b")); - Assert.assertThat(innerBuckets.query("/0/COUNT(*)/value"), equalTo(1)); - Assert.assertThat(innerBuckets.query("/1/key"), equalTo("c")); - Assert.assertThat(innerBuckets.query("/1/COUNT(*)/value"), equalTo(1)); - Assert.assertThat(innerBuckets.query("/2/key"), equalTo("zz")); - Assert.assertThat(innerBuckets.query("/2/COUNT(*)/value"), equalTo(1)); - - innerBuckets = (JSONArray)msgInfoBuckets.optQuery("/1/message.info@NESTED/message.info/buckets"); - Assert.assertThat(innerBuckets.query("/0/key"), equalTo("a")); - Assert.assertThat(innerBuckets.query("/0/COUNT(*)/value"), equalTo(2)); - Assert.assertThat(innerBuckets.query("/1/key"), equalTo("c")); - Assert.assertThat(innerBuckets.query("/1/COUNT(*)/value"), equalTo(1)); - } - - @Test - public void countAggWithoutWhere() throws IOException { - String sql = "SELECT e.name, COUNT(p) as c " + - "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + - "GROUP BY e.name " + - "HAVING c > 1"; - - JSONObject result = executeQuery(sql); - JSONObject aggregation = getAggregation(result, "name.keyword"); - JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); - - Assert.assertNotNull(bucket); - Assert.assertThat(bucket.length(), equalTo(2)); - Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); - Assert.assertThat(bucket.query("/0/projects@NESTED/c/value"), equalTo(3)); - Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); - Assert.assertThat(bucket.query("/1/projects@NESTED/c/value"), equalTo(2)); - } - - @Test - public void countAggWithWhereOnParent() throws IOException { - String sql = "SELECT e.name, COUNT(p) as c " + - "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + - "WHERE e.name like '%smith%' " + - "GROUP BY e.name " + - "HAVING c > 1"; - - JSONObject result = executeQuery(sql); - JSONObject aggregation = getAggregation(result, "name.keyword"); - JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); - - Assert.assertNotNull(bucket); - Assert.assertThat(bucket.length(), equalTo(2)); - Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); - Assert.assertThat(bucket.query("/0/projects@NESTED/c/value"), equalTo(3)); - Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); - Assert.assertThat(bucket.query("/1/projects@NESTED/c/value"), equalTo(2)); - } - - @Test - public void countAggWithWhereOnNested() throws IOException { - String sql = "SELECT e.name, COUNT(p) as c " + - "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + - "WHERE p.name LIKE '%security%' " + - "GROUP BY e.name " + - "HAVING c > 1"; - - JSONObject result = executeQuery(sql); - JSONObject aggregation = getAggregation(result, "name.keyword"); - JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); - - Assert.assertNotNull(bucket); - Assert.assertThat(bucket.length(), equalTo(2)); - Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); - Assert.assertThat(bucket.query("/0/projects@NESTED/projects@FILTER/c/value"), equalTo(2)); - Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); - Assert.assertThat(bucket.query("/1/projects@NESTED/projects@FILTER/c/value"), equalTo(2)); - } - - @Test - public void countAggWithWhereOnParentOrNested() throws IOException { - String sql = "SELECT e.name, COUNT(p) as c " + - "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + - "WHERE e.name like '%smith%' or p.name LIKE '%security%' " + - "GROUP BY e.name " + - "HAVING c > 1"; - - JSONObject result = executeQuery(sql); - JSONObject aggregation = getAggregation(result, "name.keyword"); - JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); - - Assert.assertNotNull(bucket); - Assert.assertThat(bucket.length(), equalTo(2)); - Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); - Assert.assertThat(bucket.query("/0/projects@NESTED/c/value"), equalTo(3)); - Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); - Assert.assertThat(bucket.query("/1/projects@NESTED/c/value"), equalTo(2)); - } - - @Test - public void countAggWithWhereOnParentAndNested() throws IOException { - String sql = "SELECT e.name, COUNT(p) as c " + - "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + - "WHERE e.name like '%smith%' AND p.name LIKE '%security%' " + - "GROUP BY e.name " + - "HAVING c > 1"; - - JSONObject result = executeQuery(sql); - JSONObject aggregation = getAggregation(result, "name.keyword"); - JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); - - Assert.assertNotNull(bucket); - Assert.assertThat(bucket.length(), equalTo(2)); - Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); - Assert.assertThat(bucket.query("/0/projects@NESTED/projects@FILTER/c/value"), equalTo(2)); - Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); - Assert.assertThat(bucket.query("/1/projects@NESTED/projects@FILTER/c/value"), equalTo(2)); - } - - @Test - public void countAggWithWhereOnNestedAndNested() throws IOException { - String sql = "SELECT e.name, COUNT(p) as c " + - "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + - "WHERE p.started_year > 2000 AND p.name LIKE '%security%' " + - "GROUP BY e.name " + - "HAVING c > 0"; - - JSONObject result = executeQuery(sql); - JSONObject aggregation = getAggregation(result, "name.keyword"); - JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); - - Assert.assertNotNull(bucket); - Assert.assertThat(bucket.length(), equalTo(2)); - Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); - Assert.assertThat(bucket.query("/0/projects@NESTED/projects@FILTER/c/value"), equalTo(1)); - Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); - Assert.assertThat(bucket.query("/1/projects@NESTED/projects@FILTER/c/value"), equalTo(1)); - } - - @Test - public void countAggWithWhereOnNestedOrNested() throws IOException { - String sql = "SELECT e.name, COUNT(p) as c " + - "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + - "WHERE p.started_year > 2000 OR p.name LIKE '%security%' " + - "GROUP BY e.name " + - "HAVING c > 1"; - - JSONObject result = executeQuery(sql); - JSONObject aggregation = getAggregation(result, "name.keyword"); - JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); - - Assert.assertNotNull(bucket); - Assert.assertThat(bucket.length(), equalTo(2)); - Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); - Assert.assertThat(bucket.query("/0/projects@NESTED/projects@FILTER/c/value"), equalTo(2)); - Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); - Assert.assertThat(bucket.query("/1/projects@NESTED/projects@FILTER/c/value"), equalTo(2)); - } - - @Test - public void countAggOnNestedInnerFieldWithoutWhere() throws IOException { - String sql = "SELECT e.name, COUNT(p.started_year) as count " + - "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + - "WHERE p.name LIKE '%security%' " + - "GROUP BY e.name " + - "HAVING count > 0"; - - JSONObject result = executeQuery(sql); - JSONObject aggregation = getAggregation(result, "name.keyword"); - JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); - - Assert.assertNotNull(bucket); - Assert.assertThat(bucket.length(), equalTo(2)); - Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); - Assert.assertThat( - bucket.query("/0/projects.started_year@NESTED/projects.started_year@FILTER/count/value"), - equalTo(2)); - Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); - Assert.assertThat( - bucket.query("/1/projects.started_year@NESTED/projects.started_year@FILTER/count/value"), - equalTo(2)); - } - - @Test - public void maxAggOnNestedInnerFieldWithoutWhere() throws IOException { - String sql = "SELECT e.name, MAX(p.started_year) as max " + - "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + - "WHERE p.name LIKE '%security%' " + - "GROUP BY e.name"; - - JSONObject result = executeQuery(sql); - JSONObject aggregation = getAggregation(result, "name.keyword"); - JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); - - Assert.assertNotNull(bucket); - Assert.assertThat(bucket.length(), equalTo(2)); - Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); - Assert.assertThat( - bucket.query("/0/projects.started_year@NESTED/projects.started_year@FILTER/max/value"), - equalTo(2015.0)); - Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); - Assert.assertThat( - bucket.query("/1/projects.started_year@NESTED/projects.started_year@FILTER/max/value"), - equalTo(2015.0)); - } - - @Test - public void havingCountAggWithoutWhere() throws IOException { - String sql = "SELECT e.name " + - "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + - "GROUP BY e.name " + - "HAVING COUNT(p) > 1"; - - JSONObject result = executeQuery(sql); - JSONObject aggregation = getAggregation(result, "name.keyword"); - JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); - - Assert.assertNotNull(bucket); - Assert.assertThat(bucket.length(), equalTo(2)); - Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); - Assert.assertThat(bucket.query("/0/projects@NESTED/count_0/value"), equalTo(3)); - Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); - Assert.assertThat(bucket.query("/1/projects@NESTED/count_0/value"), equalTo(2)); - } - - @Test - public void havingCountAggWithWhereOnParent() throws IOException { - String sql = "SELECT e.name " + - "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + - "WHERE e.name like '%smith%' " + - "GROUP BY e.name " + - "HAVING COUNT(p) > 1"; - - JSONObject result = executeQuery(sql); - JSONObject aggregation = getAggregation(result, "name.keyword"); - JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); - - Assert.assertNotNull(bucket); - Assert.assertThat(bucket.length(), equalTo(2)); - Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); - Assert.assertThat(bucket.query("/0/projects@NESTED/count_0/value"), equalTo(3)); - Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); - Assert.assertThat(bucket.query("/1/projects@NESTED/count_0/value"), equalTo(2)); - } - - @Test - public void havingCountAggWithWhereOnNested() throws IOException { - String sql = "SELECT e.name " + - "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + - "WHERE p.name LIKE '%security%' " + - "GROUP BY e.name " + - "HAVING COUNT(p) > 1"; - - JSONObject result = executeQuery(sql); - JSONObject aggregation = getAggregation(result, "name.keyword"); - JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); - - Assert.assertNotNull(bucket); - Assert.assertThat(bucket.length(), equalTo(2)); - Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); - Assert.assertThat(bucket.query("/0/projects@NESTED/projects@FILTER/count_0/value"), equalTo(2)); - Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); - Assert.assertThat(bucket.query("/1/projects@NESTED/projects@FILTER/count_0/value"), equalTo(2)); - } - - @Test - public void havingCountAggWithWhereOnParentOrNested() throws IOException { - String sql = "SELECT e.name " + - "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + - "WHERE e.name like '%smith%' or p.name LIKE '%security%' " + - "GROUP BY e.name " + - "HAVING COUNT(p) > 1"; - - JSONObject result = executeQuery(sql); - JSONObject aggregation = getAggregation(result, "name.keyword"); - JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); - - Assert.assertNotNull(bucket); - Assert.assertThat(bucket.length(), equalTo(2)); - Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); - Assert.assertThat(bucket.query("/0/projects@NESTED/count_0/value"), equalTo(3)); - Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); - Assert.assertThat(bucket.query("/1/projects@NESTED/count_0/value"), equalTo(2)); - } - - @Test - public void havingCountAggWithWhereOnParentAndNested() throws IOException { - String sql = "SELECT e.name " + - "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + - "WHERE e.name like '%smith%' AND p.name LIKE '%security%' " + - "GROUP BY e.name " + - "HAVING COUNT(p) > 1"; - - JSONObject result = executeQuery(sql); - JSONObject aggregation = getAggregation(result, "name.keyword"); - JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); - - Assert.assertNotNull(bucket); - Assert.assertThat(bucket.length(), equalTo(2)); - Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); - Assert.assertThat(bucket.query("/0/projects@NESTED/projects@FILTER/count_0/value"), equalTo(2)); - Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); - Assert.assertThat(bucket.query("/1/projects@NESTED/projects@FILTER/count_0/value"), equalTo(2)); - } - - @Test - public void havingCountAggWithWhereOnNestedAndNested() throws IOException { - String sql = "SELECT e.name " + - "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + - "WHERE p.started_year > 2000 AND p.name LIKE '%security%' " + - "GROUP BY e.name " + - "HAVING COUNT(p) > 0"; - - JSONObject result = executeQuery(sql); - JSONObject aggregation = getAggregation(result, "name.keyword"); - JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); - - Assert.assertNotNull(bucket); - Assert.assertThat(bucket.length(), equalTo(2)); - Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); - Assert.assertThat(bucket.query("/0/projects@NESTED/projects@FILTER/count_0/value"), equalTo(1)); - Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); - Assert.assertThat(bucket.query("/1/projects@NESTED/projects@FILTER/count_0/value"), equalTo(1)); - } - - @Test - public void havingCountAggWithWhereOnNestedOrNested() throws IOException { - String sql = "SELECT e.name " + - "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + - "WHERE p.started_year > 2000 OR p.name LIKE '%security%' " + - "GROUP BY e.name " + - "HAVING COUNT(p) > 1"; - - JSONObject result = executeQuery(sql); - JSONObject aggregation = getAggregation(result, "name.keyword"); - JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); - - Assert.assertNotNull(bucket); - Assert.assertThat(bucket.length(), equalTo(2)); - Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); - Assert.assertThat(bucket.query("/0/projects@NESTED/projects@FILTER/count_0/value"), equalTo(2)); - Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); - Assert.assertThat(bucket.query("/1/projects@NESTED/projects@FILTER/count_0/value"), equalTo(2)); - } - - @Test - public void havingCountAggOnNestedInnerFieldWithoutWhere() throws IOException { - String sql = "SELECT e.name " + - "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + - "WHERE p.name LIKE '%security%' " + - "GROUP BY e.name " + - "HAVING COUNT(p.started_year) > 0"; - - JSONObject result = executeQuery(sql); - JSONObject aggregation = getAggregation(result, "name.keyword"); - JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); - - Assert.assertNotNull(bucket); - Assert.assertThat(bucket.length(), equalTo(2)); - Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); - Assert.assertThat( - bucket.query("/0/projects.started_year@NESTED/projects.started_year@FILTER/count_0/value"), - equalTo(2)); - Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); - Assert.assertThat( - bucket.query("/1/projects.started_year@NESTED/projects.started_year@FILTER/count_0/value"), - equalTo(2)); - } - - @Test - public void havingMaxAggOnNestedInnerFieldWithoutWhere() throws IOException { - String sql = "SELECT e.name " + - "FROM elasticsearch-sql_test_index_employee_nested AS e, e.projects AS p " + - "WHERE p.name LIKE '%security%' " + - "GROUP BY e.name " + - "HAVING MAX(p.started_year) > 1990"; - - JSONObject result = executeQuery(sql); - JSONObject aggregation = getAggregation(result, "name.keyword"); - JSONArray bucket = (JSONArray) aggregation.optQuery("/buckets"); - - Assert.assertNotNull(bucket); - Assert.assertThat(bucket.length(), equalTo(2)); - Assert.assertThat(bucket.query("/0/key"), equalTo("Bob Smith")); - Assert.assertThat( - bucket.query("/0/projects.started_year@NESTED/projects.started_year@FILTER/max_0/value"), - equalTo(2015.0)); - Assert.assertThat(bucket.query("/1/key"), equalTo("Jane Smith")); - Assert.assertThat( - bucket.query("/1/projects.started_year@NESTED/projects.started_year@FILTER/max_0/value"), - equalTo(2015.0)); - } - - /*********************************************************** - Matchers for Non-Aggregation Testing - ***********************************************************/ - - @SafeVarargs - private final Matcher hits(Matcher... subMatchers) { - return featureValueOf("hits", arrayContainingInAnyOrder(subMatchers), resp -> resp.getHits().getHits()); - } - - @SafeVarargs - private final Matcher hit(Matcher... subMatchers) { - return allOf(subMatchers); - } - - private Matcher myNum(int value) { - return kv("myNum", is(value)); - } - - private Matcher myNum(int[] values) { - - return new BaseMatcher() { - - @Override - public boolean matches(Object item) { - - if (item instanceof SearchHit) { - final SearchHit hit = (SearchHit) item; - ArrayList actualValues = (ArrayList) hit.getSourceAsMap().get("myNum"); - - if (actualValues.size() != values.length) { - return false; - } - for (int i = 0; i < values.length; ++i) { - if (values[i] != actualValues.get(i)) { - return false; - } - } - return true; - } - - return false; - } - - @Override - public void describeTo(Description description) { - } - }; - } - - private Matcher someField(String value) { - return kv("someField", is(value)); - } - - private Matcher author(String value) { - return kv("author", is(value)); - } - - private Matcher info(String value) { - return kv("info", is(value)); - } - - private Matcher kv(String key, Matcher valMatcher) { - return featureValueOf(key, valMatcher, hit -> hit.getSourceAsMap().get(key)); - } - - @SafeVarargs - private final Matcher innerHits(String path, Matcher... innerHitMatchers) { - return featureValueOf( - "innerHits", - arrayContainingInAnyOrder(innerHitMatchers), - hit -> hit.getInnerHits().get(path).getHits() - ); - } - - /*********************************************************** - Matchers for Aggregation Testing - ***********************************************************/ - - private FeatureMatcher featureValueOf(String name, Matcher subMatcher, Function getter) { - return new FeatureMatcher(subMatcher, name, name) { - @Override - protected U featureValueOf(T actual) { - return getter.apply(actual); - } - }; - } - - /*********************************************************** - Query Utility to Fetch Response for SQL - ***********************************************************/ - - private SearchResponse query(String select, String... statements) throws IOException { - return execute(select + " " + FROM + " " + String.join(" ", statements)); - } - - private SearchResponse execute(String sql) throws IOException { - final JSONObject jsonObject = executeQuery(sql); - - final XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser( - NamedXContentRegistry.EMPTY, - LoggingDeprecationHandler.INSTANCE, - jsonObject.toString()); - return SearchResponse.fromXContent(parser); - } - - private JSONObject getAggregation(final JSONObject queryResult, final String aggregationName) - { - final String aggregationsObjName = "aggregations"; - Assert.assertTrue(queryResult.has(aggregationsObjName)); - - final JSONObject aggregations = queryResult.getJSONObject(aggregationsObjName); - Assert.assertTrue(aggregations.has(aggregationName)); - return aggregations.getJSONObject(aggregationName); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java deleted file mode 100644 index e5aa1c1b68..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import org.apache.http.Header; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.message.BasicHeader; -import org.apache.http.ssl.SSLContextBuilder; -import org.apache.http.util.EntityUtils; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestClientBuilder; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.test.rest.ESRestTestCase; -import org.json.JSONArray; -import org.json.JSONObject; - -import java.io.IOException; -import java.util.Map; -import java.util.Optional; - -/** - * ODFE integration test base class to support both security disabled and enabled ODFE cluster. - */ -public abstract class ODFERestTestCase extends ESRestTestCase { - - protected boolean isHttps() { - boolean isHttps = Optional.ofNullable(System.getProperty("https")) - .map("true"::equalsIgnoreCase).orElse(false); - if (isHttps) { - //currently only external cluster is supported for security enabled testing - if (!Optional.ofNullable(System.getProperty("tests.rest.cluster")).isPresent()) { - throw new RuntimeException("external cluster url should be provided for security enabled testing"); - } - } - - return isHttps; - } - - protected String getProtocol() { - return isHttps() ? "https" : "http"; - } - - protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException { - RestClientBuilder builder = RestClient.builder(hosts); - if (isHttps()) { - configureHttpsClient(builder, settings); - } else { - configureClient(builder, settings); - } - - builder.setStrictDeprecationMode(true); - return builder.build(); - } - - protected static void wipeAllODFEIndices() throws IOException { - Response response = client().performRequest(new Request("GET", "/_cat/indices?format=json")); - JSONArray jsonArray = new JSONArray(EntityUtils.toString(response.getEntity(), "UTF-8")); - for (Object object : jsonArray) { - JSONObject jsonObject = (JSONObject)object; - String indexName = jsonObject.getString("index"); - //.opendistro_security isn't allowed to delete from cluster - if (!".opendistro_security".equals(indexName)) { - client().performRequest(new Request("DELETE", "/" + indexName)); - } - } - } - - protected static void configureHttpsClient(RestClientBuilder builder, Settings settings) throws IOException { - Map headers = ThreadContext.buildDefaultHeaders(settings); - Header[] defaultHeaders = new Header[headers.size()]; - int i = 0; - for (Map.Entry entry : headers.entrySet()) { - defaultHeaders[i++] = new BasicHeader(entry.getKey(), entry.getValue()); - } - builder.setDefaultHeaders(defaultHeaders); - builder.setHttpClientConfigCallback(httpClientBuilder -> { - String userName = Optional.ofNullable(System.getProperty("user")) - .orElseThrow(() -> new RuntimeException("user name is missing")); - String password = Optional.ofNullable(System.getProperty("password")) - .orElseThrow(() -> new RuntimeException("password is missing")); - CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, password)); - try { - return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider) - //disable the certificate since our testing cluster just uses the default security configuration - .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) - .setSSLContext(SSLContextBuilder.create() - .loadTrustMaterial(null, (chains, authType) -> true) - .build()); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - - final String socketTimeoutString = settings.get(CLIENT_SOCKET_TIMEOUT); - final TimeValue socketTimeout = - TimeValue.parseTimeValue(socketTimeoutString == null ? "60s" : socketTimeoutString, CLIENT_SOCKET_TIMEOUT); - builder.setRequestConfigCallback(conf -> conf.setSocketTimeout(Math.toIntExact(socketTimeout.getMillis()))); - if (settings.hasValue(CLIENT_PATH_PREFIX)) { - builder.setPathPrefix(settings.get(CLIENT_PATH_PREFIX)); - } - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/OrderIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/OrderIT.java deleted file mode 100644 index 61dede0572..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/OrderIT.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import org.json.JSONArray; -import org.json.JSONObject; -import org.junit.Test; - -import java.io.IOException; - -import static org.hamcrest.Matchers.equalTo; - -public class OrderIT extends SQLIntegTestCase { - - @Override - protected void init() throws Exception { - loadIndex(Index.ORDER); - } - - @Test - public void simpleOrder() throws IOException { - String query = "SELECT id, name FROM elasticsearch-sql_test_index_order ORDER BY id"; - JSONArray result = getSortExplain(query); - assertThat(result.length(), equalTo(1)); - JSONObject jsonObject = getSortByField(result, "id"); - assertThat(query(result, "/0/id/order"), equalTo("asc")); - assertFalse(jsonObject.getJSONObject("id").has("missing")); - } - - @Test - public void orderByScore() throws IOException { - String query = "SELECT * FROM elasticsearch-sql_test_index_order ORDER BY _score"; - JSONArray result = getSortExplain(query); - assertThat(result.length(), equalTo(1)); - JSONObject jsonObject = getSortByField(result, "_score"); - assertThat(query(result, "/0/_score/order"), equalTo("asc")); - assertFalse(jsonObject.getJSONObject("_score").has("missing")); - - JSONObject response = executeQuery(query); - JSONArray hits = getHits(response); - assertThat(hits.length(), equalTo(7)); - } - - @Test - public void simpleOrderMultipleFields() throws IOException { - String query = "SELECT id, name FROM elasticsearch-sql_test_index_order ORDER BY id, name"; - JSONArray result = getSortExplain(query); - assertThat(result.length(), equalTo(2)); - assertTrue(result.getJSONObject(0).has("id")); - assertTrue(result.getJSONObject(1).has("name.keyword")); - } - - @Test - public void explicitOrderType() throws IOException { - String query = "SELECT id, name FROM elasticsearch-sql_test_index_order ORDER BY id ASC, name DESC"; - JSONArray result = getSortExplain(query); - assertThat(result.length(), equalTo(2)); - assertThat(query(result, "/0/id/order"), equalTo("asc")); - assertThat(query(result, "/1/name.keyword/order"), equalTo("desc")); - } - - @Test - public void orderByIsNull() throws IOException { - String query = "SELECT * FROM elasticsearch-sql_test_index_order ORDER BY id IS NULL, id DESC"; - JSONArray result = getSortExplain(query); - assertThat(result.length(), equalTo(1)); - assertThat(query(result, "/0/id/order"), equalTo("desc")); - assertThat(query(result, "/0/id/missing"), equalTo("_last")); - - JSONObject response = executeQuery(query); - JSONArray hits = getHits(response); - assertThat(query(hits, "/0/_source/id"), equalTo("5")); - - // Another equivalent syntax - assertThat(explainQuery("SELECT * FROM elasticsearch-sql_test_index_order " + - "ORDER BY id IS NULL, id DESC"), - equalTo(explainQuery("SELECT * FROM elasticsearch-sql_test_index_order " + - "ORDER BY id IS NULL DESC")) - ); - } - - @Test - public void orderByIsNotNull() throws IOException { - String query = "SELECT id, name FROM elasticsearch-sql_test_index_order ORDER BY name IS NOT NULL"; - JSONArray result = getSortExplain(query); - assertThat(1, equalTo(result.length())); - assertThat(query(result, "/0/name.keyword/order"), equalTo("asc")); - assertThat(query(result, "/0/name.keyword/missing"), equalTo("_first")); - - JSONObject response = executeQuery(query); - JSONArray hits = getHits(response); - assertFalse(hits.getJSONObject(0).getJSONObject("_source").has("name")); - assertThat(hits.getJSONObject(hits.length()-1).query("/_source/name").toString(), equalTo("f")); - - // Another equivalent syntax - assertThat(explainQuery("SELECT id, name FROM elasticsearch-sql_test_index_order " + - "ORDER BY name IS NOT NULL"), - equalTo(explainQuery("SELECT id, name FROM elasticsearch-sql_test_index_order " + - "ORDER BY name IS NOT NULL ASC")) - ); - } - - @Test - public void multipleOrderByWithNulls() throws IOException { - String query = "SELECT id, name FROM elasticsearch-sql_test_index_order ORDER BY id IS NULL, name IS NOT NULL"; - JSONArray result = getSortExplain(query); - assertThat(result.length(), equalTo(2)); - assertThat(query(result, "/0/id/missing"), equalTo("_last")); - assertThat(query(result, "/1/name.keyword/missing"), equalTo("_first")); - } - - @Test - public void testOrderByMergeForSameField() throws IOException { - String query = "SELECT * FROM elasticsearch-sql_test_index_order " + - "ORDER BY id IS NULL, name DESC, id DESC, id IS NOT NULL, name IS NULL"; - JSONArray result = getSortExplain(query); - assertThat(2, equalTo(result.length())); - assertThat(query(result, "/0/id/order"), equalTo("asc")); - assertThat(query(result, "/0/id/missing"), equalTo("_first")); - assertThat(query(result, "/1/name.keyword/order"), equalTo("asc")); - assertThat(query(result, "/1/name.keyword/missing"), equalTo("_last")); - } - - private JSONArray getSortExplain(String sqlQuery) throws IOException { - String result = explainQuery(sqlQuery); - JSONObject jsonObject = new JSONObject(result); - return jsonObject.getJSONArray("sort"); - } - - private JSONObject getSortByField(JSONArray sortArray, String orderByName) { - JSONObject jsonObject; - for(int i = 0; i < sortArray.length(); i++) { - jsonObject = (JSONObject) sortArray.get(i); - if (jsonObject.has(orderByName)) { - return jsonObject; - } - } - return null; - } - - private String query(JSONArray jsonArray, String jsonPath) { - return jsonArray.query(jsonPath).toString(); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/OrdinalAliasRewriterIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/OrdinalAliasRewriterIT.java deleted file mode 100644 index 5b7d729304..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/OrdinalAliasRewriterIT.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; -import org.junit.Test; - -import java.io.IOException; - -import static org.hamcrest.Matchers.equalTo; - -public class OrdinalAliasRewriterIT extends SQLIntegTestCase { - - @Override - protected void init() throws Exception { - loadIndex(Index.ACCOUNT); - } - - // tests query results with jdbc output - @Test - public void simpleGroupByOrdinal() { - String expected = executeQuery(StringUtils.format( - "SELECT lastname FROM %s AS b GROUP BY lastname LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); - String actual = executeQuery(StringUtils.format( - "SELECT lastname FROM %s AS b GROUP BY 1 LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); - assertThat(actual, equalTo(expected)); - } - - @Test - public void multipleGroupByOrdinal() { - String expected = executeQuery(StringUtils.format( - "SELECT lastname, firstname, age FROM %s AS b GROUP BY firstname, age, lastname LIMIT 3", - TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); - String actual = executeQuery(StringUtils.format( - "SELECT lastname, firstname, age FROM %s AS b GROUP BY 2, 3, 1 LIMIT 3", - TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); - assertThat(actual, equalTo(expected)); - } - - @Test - public void selectFieldiWithBacticksGroupByOrdinal() { - String expected = executeQuery(StringUtils.format( - "SELECT `lastname` FROM %s AS b GROUP BY `lastname` LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); - String actual = executeQuery(StringUtils.format( - "SELECT `lastname` FROM %s AS b GROUP BY 1 LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); - assertThat(actual, equalTo(expected)); - } - - @Test - public void selectFieldiWithBacticksAndTableAliasGroupByOrdinal() { - String expected = executeQuery(StringUtils.format( - "SELECT `b`.`lastname`, `age`, firstname FROM %s AS b GROUP BY `age`, `b`.`lastname` , firstname LIMIT 10", - TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); - String actual = executeQuery(StringUtils.format( - "SELECT `b`.`lastname`, `age`, firstname FROM %s AS b GROUP BY 2, 1, 3 LIMIT 10", - TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); - assertThat(actual, equalTo(expected)); - } - - @Test - public void simpleOrderByOrdinal() { - String expected = executeQuery(StringUtils.format( - "SELECT lastname FROM %s AS b ORDER BY lastname LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); - String actual = executeQuery(StringUtils.format( - "SELECT lastname FROM %s AS b ORDER BY 1 LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); - assertThat(actual, equalTo(expected)); - } - - @Test - public void multipleOrderByOrdinal() { - String expected = executeQuery(StringUtils.format( - "SELECT lastname, firstname, age FROM %s AS b ORDER BY firstname, age, lastname LIMIT 3", - TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); - String actual = executeQuery(StringUtils.format( - "SELECT lastname, firstname, age FROM %s AS b ORDER BY 2, 3, 1 LIMIT 3", - TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); - assertThat(actual, equalTo(expected)); - } - - @Test - public void selectFieldiWithBacticksOrderByOrdinal() { - String expected = executeQuery(StringUtils.format( - "SELECT `lastname` FROM %s AS b ORDER BY `lastname` LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); - String actual = executeQuery(StringUtils.format( - "SELECT `lastname` FROM %s AS b ORDER BY 1 LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); - assertThat(actual, equalTo(expected)); - } - - @Test - public void selectFieldiWithBacticksAndTableAliasOrderByOrdinal() { - String expected = executeQuery(StringUtils.format( - "SELECT `b`.`lastname` FROM %s AS b ORDER BY `b`.`lastname` LIMIT 3", - TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); - String actual = executeQuery(StringUtils.format( - "SELECT `b`.`lastname` FROM %s AS b ORDER BY 1 LIMIT 3", - TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); - assertThat(actual, equalTo(expected)); - } - - // ORDER BY IS NULL/NOT NULL - @Test - public void selectFieldiWithBacticksAndTableAliasOrderByOrdinalAndNull() { - String expected = executeQuery(StringUtils.format( - "SELECT `b`.`lastname`, age FROM %s AS b ORDER BY `b`.`lastname` IS NOT NULL DESC, age is NULL LIMIT 3", - TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); - String actual = executeQuery(StringUtils.format( - "SELECT `b`.`lastname`, age FROM %s AS b ORDER BY 1 IS NOT NULL DESC, 2 IS NULL LIMIT 3", - TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); - assertThat(actual, equalTo(expected)); - } - - - // explain - @Test - public void explainSelectFieldiWithBacticksAndTableAliasGroupByOrdinal() throws IOException { - String expected = explainQuery(StringUtils.format( - "SELECT `b`.`lastname` FROM %s AS b GROUP BY `b`.`lastname` LIMIT 3", - TestsConstants.TEST_INDEX_ACCOUNT)); - String actual = explainQuery(StringUtils.format( - "SELECT `b`.`lastname` FROM %s AS b GROUP BY 1 LIMIT 3", - TestsConstants.TEST_INDEX_ACCOUNT)); - assertThat(actual, equalTo(expected)); - } - - @Test - public void explainSelectFieldiWithBacticksAndTableAliasOrderByOrdinal() throws IOException { - String expected = explainQuery(StringUtils.format( - "SELECT `b`.`lastname` FROM %s AS b ORDER BY `b`.`lastname` LIMIT 3", - TestsConstants.TEST_INDEX_ACCOUNT)); - String actual = explainQuery(StringUtils.format( - "SELECT `b`.`lastname` FROM %s AS b ORDER BY 1 LIMIT 3", - TestsConstants.TEST_INDEX_ACCOUNT)); - assertThat(actual, equalTo(expected)); - } - - // explain ORDER BY IS NULL/NOT NULL - @Test - public void explainSelectFieldiWithBacticksAndTableAliasOrderByOrdinalAndNull() throws IOException { - String expected = explainQuery(StringUtils.format( - "SELECT `b`.`lastname`, age FROM %s AS b ORDER BY `b`.`lastname` IS NOT NULL DESC, age is NULL LIMIT 3", - TestsConstants.TEST_INDEX_ACCOUNT)); - String actual = explainQuery(StringUtils.format( - "SELECT `b`.`lastname`, age FROM %s AS b ORDER BY 1 IS NOT NULL DESC, 2 IS NULL LIMIT 3", - TestsConstants.TEST_INDEX_ACCOUNT)); - assertThat(actual, equalTo(expected)); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/PluginIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/PluginIT.java deleted file mode 100644 index 561072cbed..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/PluginIT.java +++ /dev/null @@ -1,451 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import org.elasticsearch.client.Request; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; -import org.json.JSONObject; -import org.junit.Test; - -import java.io.IOException; -import java.util.Locale; - -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getResponseBody; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_ACCOUNT; -import static org.hamcrest.Matchers.equalTo; - -public class PluginIT extends SQLIntegTestCase { - - @Override - protected void init() throws Exception { - wipeAllClusterSettings(); - } - - @Test - public void sqlEnableSettingsTest() throws IOException { - loadIndex(Index.ACCOUNT); - updateClusterSettings(new ClusterSetting(PERSISTENT, "opendistro.sql.enabled", "true")); - String query = String.format(Locale.ROOT, "SELECT firstname FROM %s WHERE account_number=1", TEST_INDEX_ACCOUNT); - JSONObject queryResult = executeQuery(query); - assertThat(getHits(queryResult).length(), equalTo(1)); - - updateClusterSettings(new ClusterSetting(PERSISTENT, "opendistro.sql.enabled", "false")); - Response response = null; - try { - queryResult = executeQuery(query); - } catch (ResponseException ex) { - response = ex.getResponse(); - } - - queryResult = new JSONObject(TestUtils.getResponseBody(response)); - assertThat(queryResult.getInt("status"), equalTo(400)); - JSONObject error = queryResult.getJSONObject("error"); - assertThat(error.getString("reason"), equalTo("Invalid SQL query")); - assertThat(error.getString("details"), equalTo("Either opendistro.sql.enabled or rest.action.multi.allow_explicit_index setting is false")); - assertThat(error.getString("type"), equalTo("SQLFeatureDisabledException")); - wipeAllClusterSettings(); - } - - @Test - public void sqlTransientOnlySettingTest() throws IOException { - // (1) compact form - String settings = "{" + - " \"transient\": {" + - " \"opendistro.sql.metrics.rollinginterval\": \"80\"" + - " }" + - "}"; - JSONObject actual = updateViaSQLSettingsAPI(settings); - JSONObject expected = new JSONObject("{" + - " \"acknowledged\" : true," + - " \"persistent\" : { }," + - " \"transient\" : {" + - " \"opendistro\" : {" + - " \"sql\" : {" + - " \"metrics\" : {" + - " \"rollinginterval\" : \"80\"" + - " }" + - " }" + - " }" + - " }" + - "}"); - assertTrue(actual.similar(expected)); - - // (2) partial expanded form - settings = "{" + - " \"transient\": {" + - " \"opendistro\" : {" + - " \"sql\" : {" + - " \"metrics.rollinginterval\": \"75\"" + - " }" + - " }" + - " }" + - "}"; - actual = updateViaSQLSettingsAPI(settings); - expected = new JSONObject("{" + - " \"acknowledged\" : true," + - " \"persistent\" : { }," + - " \"transient\" : {" + - " \"opendistro\" : {" + - " \"sql\" : {" + - " \"metrics\" : {" + - " \"rollinginterval\" : \"75\"" + - " }" + - " }" + - " }" + - " }" + - "}"); - assertTrue(actual.similar(expected)); - - - // (3) full expanded form - settings = "{" + - " \"transient\": {" + - " \"opendistro\" : {" + - " \"sql\" : {" + - " \"metrics\": {" + - " \"rollinginterval\": \"65\"" + - " }" + - " }" + - " }" + - " }" + - "}"; - actual = updateViaSQLSettingsAPI(settings); - expected = new JSONObject("{" + - " \"acknowledged\" : true," + - " \"persistent\" : { }," + - " \"transient\" : {" + - " \"opendistro\" : {" + - " \"sql\" : {" + - " \"metrics\" : {" + - " \"rollinginterval\" : \"65\"" + - " }" + - " }" + - " }" + - " }" + - "}"); - assertTrue(actual.similar(expected)); - } - - @Test - public void sqlPersistentOnlySettingTest() throws IOException { - // (1) compact form - String settings = "{" + - " \"persistent\": {" + - " \"opendistro.sql.metrics.rollinginterval\": \"80\"" + - " }" + - "}"; - JSONObject actual = updateViaSQLSettingsAPI(settings); - JSONObject expected = new JSONObject("{" + - " \"acknowledged\" : true," + - " \"transient\" : { }," + - " \"persistent\" : {" + - " \"opendistro\" : {" + - " \"sql\" : {" + - " \"metrics\" : {" + - " \"rollinginterval\" : \"80\"" + - " }" + - " }" + - " }" + - " }" + - "}"); - assertTrue(actual.similar(expected)); - - // (2) partial expanded form - settings = "{" + - " \"persistent\": {" + - " \"opendistro\" : {" + - " \"sql\" : {" + - " \"metrics.rollinginterval\": \"75\"" + - " }" + - " }" + - " }" + - "}"; - actual = updateViaSQLSettingsAPI(settings); - expected = new JSONObject("{" + - " \"acknowledged\" : true," + - " \"transient\" : { }," + - " \"persistent\" : {" + - " \"opendistro\" : {" + - " \"sql\" : {" + - " \"metrics\" : {" + - " \"rollinginterval\" : \"75\"" + - " }" + - " }" + - " }" + - " }" + - "}"); - assertTrue(actual.similar(expected)); - - - // (3) full expanded form - settings = "{" + - " \"persistent\": {" + - " \"opendistro\" : {" + - " \"sql\" : {" + - " \"metrics\": {" + - " \"rollinginterval\": \"65\"" + - " }" + - " }" + - " }" + - " }" + - "}"; - actual = updateViaSQLSettingsAPI(settings); - expected = new JSONObject("{" + - " \"acknowledged\" : true," + - " \"transient\" : { }," + - " \"persistent\" : {" + - " \"opendistro\" : {" + - " \"sql\" : {" + - " \"metrics\" : {" + - " \"rollinginterval\" : \"65\"" + - " }" + - " }" + - " }" + - " }" + - "}"); - assertTrue(actual.similar(expected)); - } - - /** - * Both transient and persistent settings are applied for same settings. - * This is similiar to _cluster/settings behavior - */ - @Test - public void sqlCombinedSettingTest() throws IOException { - String settings = "{" + - " \"transient\": {" + - " \"opendistro.sql.metrics.rollingwindow\": \"3700\"," + - " \"opendistro.sql.query.analysis.semantic.suggestion\" : \"false\"" + - " }," + - " \"persistent\": {" + - " \"opendistro.sql.query.analysis.semantic.suggestion\" : \"true\"" + - " }" + - "}" ; - JSONObject actual = updateViaSQLSettingsAPI(settings); - JSONObject expected = new JSONObject("{" + - " \"acknowledged\" : true," + - " \"persistent\" : {" + - " \"opendistro\" : {" + - " \"sql\" : {" + - " \"query\" : {" + - " \"analysis\" : {" + - " \"semantic\" : {" + - " \"suggestion\" : \"true\"" + - " }" + - " }" + - " }" + - " }" + - " }" + - " }," + - " \"transient\" : {" + - " \"opendistro\" : {" + - " \"sql\" : {" + - " \"metrics\" : {" + - " \"rollingwindow\" : \"3700\"" + - " }," + - " \"query\" : {" + - " \"analysis\" : {" + - " \"semantic\" : {" + - " \"suggestion\" : \"false\"" + - " }" + - " }" + - " }" + - " }" + - " }" + - " }" + - "}"); - assertTrue(actual.similar(expected)); - } - - /** - * Ignore all non opendistro.sql settings. - * Only settings starting with opendistro.sql. are affected - */ - @Test - public void ignoreNonSQLSettingsTest() throws IOException { - String settings = "{" + - " \"transient\": {" + - " \"opendistro.sql.metrics.rollingwindow\": \"3700\"," + - " \"opendistro.alerting.metrics.rollingwindow\": \"3700\"," + - " \"search.max_buckets\": \"10000\"," + - " \"search.max_keep_alive\": \"24h\"" + - " }," + - " \"persistent\": {" + - " \"opendistro.sql.query.analysis.semantic.suggestion\": \"true\"," + - " \"opendistro.alerting.metrics.rollingwindow\": \"3700\"," + - " \"thread_pool.analyze.queue_size\": \"16\"" + - " }" + - "}" ; - JSONObject actual = updateViaSQLSettingsAPI(settings); - JSONObject expected = new JSONObject("{" + - " \"acknowledged\" : true," + - " \"persistent\" : {" + - " \"opendistro\" : {" + - " \"sql\" : {" + - " \"query\" : {" + - " \"analysis\" : {" + - " \"semantic\" : {" + - " \"suggestion\" : \"true\"" + - " }" + - " }" + - " }" + - " }" + - " }" + - " }," + - " \"transient\" : {" + - " \"opendistro\" : {" + - " \"sql\" : {" + - " \"metrics\" : {" + - " \"rollingwindow\" : \"3700\"" + - " }" + - " }" + - " }" + - " }" + - "}"); - assertTrue(actual.similar(expected)); - } - - @Test - public void ignoreNonTransientNonPersistentSettingsTest() throws IOException { - String settings = "{" + - " \"transient\": {" + - " \"opendistro.sql.query.response.format\": \"jdbc\"" + - " }," + - " \"persistent\": {" + - " \"opendistro.sql.query.slowlog\": \"2\"" + - " }," + - " \"hello\": {" + - " \"world\" : {" + - " \"name\" : \"John Doe\"" + - " }" + - " }" + - "}" ; - JSONObject actual = updateViaSQLSettingsAPI(settings); - JSONObject expected = new JSONObject("{" + - " \"acknowledged\" : true," + - " \"persistent\" : {" + - " \"opendistro\" : {" + - " \"sql\" : {" + - " \"query\" : {" + - " \"slowlog\" : \"2\"" + - " }" + - " }" + - " }" + - " }," + - " \"transient\" : {" + - " \"opendistro\" : {" + - " \"sql\" : {" + - " \"query\" : {" + - " \"response\" : {" + - " \"format\" : \"jdbc\"" + - " }" + - " }" + - " }" + - " }" + - " }" + - "}"); - assertTrue(actual.similar(expected)); - } - - @Test - public void sqlCombinedMixedSettingTest() throws IOException { - String settings = "{" + - " \"transient\": {" + - " \"opendistro.sql.query.response.format\": \"json\"" + - " }," + - " \"persistent\": {" + - " \"opendistro\": {" + - " \"sql\": {" + - " \"query\": {" + - " \"slowlog\": \"1\"," + - " \"response.format\": \"jdbc\"" + - " }" + - " }" + - " }" + - " }," + - " \"hello\": {" + - " \"world\": {" + - " \"city\": \"Seattle\"" + - " }" + - " }" + - "}" ; - JSONObject actual = updateViaSQLSettingsAPI(settings); - JSONObject expected = new JSONObject("{" + - " \"acknowledged\" : true," + - " \"persistent\" : {" + - " \"opendistro\" : {" + - " \"sql\" : {" + - " \"query\" : {" + - " \"slowlog\" : \"1\"," + - " \"response\" : {" + - " \"format\" : \"jdbc\"" + - " }" + - " }" + - " }" + - " }" + - " }," + - " \"transient\" : {" + - " \"opendistro\" : {" + - " \"sql\" : {" + - " \"query\" : {" + - " \"response\" : {" + - " \"format\" : \"json\"" + - " }" + - " }" + - " }" + - " }" + - " }" + - "}"); - assertTrue(actual.similar(expected)); - } - - @Test - public void nonRegisteredSQLSettingsThrowException() throws IOException { - String settings = "{" + - " \"transient\": {" + - " \"opendistro.sql.query.state.city\": \"Seattle\"" + - " }" + - "}" ; - - JSONObject actual; - Response response = null; - try { - actual = updateViaSQLSettingsAPI(settings); - } catch (ResponseException ex) { - response = ex.getResponse(); - } - - actual = new JSONObject(TestUtils.getResponseBody(response)); - assertThat(actual.getInt("status"), equalTo(400)); - assertThat(actual.query("/error/type"), equalTo("illegal_argument_exception")); - assertThat( - actual.query("/error/reason"), - equalTo("transient setting [opendistro.sql.query.state.city], not recognized") - ); - } - - protected static JSONObject updateViaSQLSettingsAPI(String body) throws IOException { - Request request = new Request("PUT", "/_opendistro/_sql/settings"); - request.setJsonEntity(body); - RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); - restOptionsBuilder.addHeader("Content-Type", "application/json"); - request.setOptions(restOptionsBuilder); - Response response = client().performRequest(request); - return new JSONObject(getResponseBody(response)); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/PreparedStatementIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/PreparedStatementIT.java deleted file mode 100644 index 647c16522e..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/PreparedStatementIT.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import org.json.JSONArray; -import org.json.JSONObject; -import org.junit.Assert; -import org.junit.Test; - -import java.io.IOException; - -// Refer to https://www.elastic.co/guide/en/elasticsearch/reference/6.5/integration-tests.html -// for detailed ESIntegTestCase usages doc. -public class PreparedStatementIT extends SQLIntegTestCase { - - @Override - protected void init() throws Exception { - loadIndex(Index.ACCOUNT); - } - - @Test - public void testPreparedStatement() throws IOException { - int ageToCompare = 35; - - JSONObject response = executeRequest(String.format("{\n" + - " \"query\": \"SELECT * FROM %s WHERE age > ? AND state in (?, ?) LIMIT ?\",\n" + - " \"parameters\": [\n" + - " {\n" + - " \"type\": \"integer\",\n" + - " \"value\": \"" + ageToCompare + "\"\n" + - " },\n" + - " {\n" + - " \"type\": \"string\",\n" + - " \"value\": \"TN\"\n" + - " },\n" + - " {\n" + - " \"type\": \"string\",\n" + - " \"value\": \"UT\"\n" + - " },\n" + - " {\n" + - " \"type\": \"integer\",\n" + - " \"value\": \"20\"\n" + - " }\n" + - " ]\n" + - "}", TestsConstants.TEST_INDEX_ACCOUNT)); - - Assert.assertTrue(response.has("hits")); - Assert.assertTrue(response.getJSONObject("hits").has("hits")); - - JSONArray hits = response.getJSONObject("hits").getJSONArray("hits"); - Assert.assertTrue(hits.length() > 0); - for(int i = 0; i ageToCompare); - } - } - - /* currently the integ test case will fail if run using Intellj, have to run using gradle command - * because the integ test cluster created by IntellJ has http diabled, need to spend some time later to - * figure out how to configure the integ test cluster properly. Related online resources: - * https://discuss.elastic.co/t/http-enabled-with-esintegtestcase/102032 - * https://discuss.elastic.co/t/help-with-esintegtestcase/105245 - @Override - protected Collection> nodePlugins() { - return Arrays.asList(MockTcpTransportPlugin.class); - } - - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder().put(super.nodeSettings(nodeOrdinal)) - // .put("node.mode", "network") - .put("http.enabled", true) - //.put("http.type", "netty4") - .build(); - } - */ -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/PrettyFormatResponseIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/PrettyFormatResponseIT.java deleted file mode 100644 index 97aa3e5a07..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/PrettyFormatResponseIT.java +++ /dev/null @@ -1,602 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import com.google.common.collect.Sets; -import org.elasticsearch.client.Request; -import org.json.JSONArray; -import org.json.JSONObject; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.stream.Stream; - -import static java.util.stream.Collectors.toSet; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; - -/** - * PrettyFormatResponseIT will likely be excluding some of the tests written in PrettyFormatResponseTest since - * those tests were asserting on class objects directly. These updated tests will only be making assertions based - * on the REST response. - * - * Any integ tests from PrettyFormatResponseTest that were excluded can perhaps later be changed and moved over - * to be unit tests. - * - * Tests from original integ tests excluded: - * - noIndexType() - * - withIndexType() - */ -public class PrettyFormatResponseIT extends SQLIntegTestCase { - - private static final Set allAccountFields = Sets.newHashSet( - "account_number", "balance", "firstname", "lastname", "age", "gender", "address", "employer", - "email", "city", "state" - ); - - private static final Set regularFields = Sets.newHashSet("someField", "myNum"); - - private static final Set messageFields = Sets.newHashSet( - "message.dayOfWeek", "message.info", "message.author"); - - private static final Set commentFields = Sets.newHashSet("comment.data", "comment.likes"); - - private static final List nameFields = Arrays.asList("firstname", "lastname"); - - private final int RESPONSE_DEFAULT_MAX_SIZE = 200; - - @Override - protected void init() throws Exception { - loadIndex(Index.ACCOUNT); - loadIndex(Index.PHRASE); - loadIndex(Index.GAME_OF_THRONES); - loadIndex(Index.NESTED); - } - - @Override - protected Request getSqlRequest(String request, boolean explain) { - Request sqlRequest = super.getSqlRequest(request, explain); - sqlRequest.addParameter("format", "jdbc"); - - return sqlRequest; - } - - @Ignore("Index type is removed in ES 7+") - @Test - public void wrongIndexType() throws IOException { - String type = "wrongType"; - try { - executeQuery(String.format(Locale.ROOT, "SELECT * FROM %s/%s", - TestsConstants.TEST_INDEX_ACCOUNT, type)); - } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), is(String.format(Locale.ROOT, "Index type %s does not exist", type))); - } - } - - @Test - public void selectAll() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * FROM %s", - TestsConstants.TEST_INDEX_ACCOUNT)); - - // This also tests that .keyword fields are ignored when SELECT * is called - assertContainsColumnsInAnyOrder(getSchema(response), allAccountFields); - assertContainsData(getDataRows(response), allAccountFields); - } - - @Test - public void selectNames() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT firstname, lastname FROM %s", - TestsConstants.TEST_INDEX_ACCOUNT)); - - assertContainsColumns(getSchema(response), nameFields); - assertContainsData(getDataRows(response), nameFields); - } - - @Ignore("Semantic analysis takes care of this") - @Test - public void selectWrongField() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT wrongField FROM %s", - TestsConstants.TEST_INDEX_ACCOUNT)); - - assertThat(getSchema(response).length(), equalTo(0)); - - // DataRows object will still get populated with SearchHits but since wrongField is not available in the Map - // each row in the response will be empty - // TODO Perhaps a code change should be made to format logic to ensure a - // 'datarows' length of 0 in response for this case - assertThat(getDataRows(response).length(), equalTo(RESPONSE_DEFAULT_MAX_SIZE)); - } - - @Test - public void selectKeyword() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT firstname.keyword FROM %s", - TestsConstants.TEST_INDEX_ACCOUNT)); - - List fields = Collections.singletonList("firstname.keyword"); - assertContainsColumns(getSchema(response), fields); - - /* - * firstname.keyword will appear in Schema but because there is no 'firstname.keyword' in SearchHits source - * the DataRows will output null. - * - * Looks like x-pack adds this keyword field to "docvalue_fields", this is likely how it ends up in SearchHits - */ - // assertContainsData(getDataRows(protocol), fields); - } - - @Test - public void selectScore() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT _score FROM %s WHERE balance > 30000", - TestsConstants.TEST_INDEX_ACCOUNT)); - - List fields = Collections.singletonList("_score"); - assertContainsColumns(getSchema(response), fields); - assertContainsData(getDataRows(response), fields); - } - - @Test - public void selectAllFromNestedWithoutFieldInFrom() throws IOException { - assertNestedFieldQueryResultContainsColumnsAndData("SELECT * FROM %s", - regularFields, fields("message", "comment")); - } - - @Test - public void selectAllFromNestedWithFieldInFrom() throws IOException { - assertNestedFieldQueryResultContainsColumnsAndData("SELECT * FROM %s e, e.message m", - regularFields, messageFields); - } - - @Test - public void selectAllFromNestedWithMultipleFieldsInFrom() throws IOException { - assertNestedFieldQueryResultContainsColumnsAndData("SELECT * FROM %s e, e.message m, e.comment c", - regularFields, messageFields, commentFields); - } - - @Test - public void selectAllNestedFromNestedWithFieldInFrom() throws IOException { - assertNestedFieldQueryResultContainsColumnsAndData("SELECT m.* FROM %s e, e.message m", messageFields); - } - - @Test - public void selectSpecificRegularFieldAndAllFromNestedWithFieldInFrom() throws IOException { - assertNestedFieldQueryResultContainsColumnsAndData("SELECT e.someField, m.* FROM %s e, e.message m", - fields("someField"), messageFields); - } - - /** - * Execute the query against index with nested fields and assert result contains columns and data as expected. - */ - @SafeVarargs - private final void assertNestedFieldQueryResultContainsColumnsAndData(String query, - Set... expectedFieldNames) throws IOException { - JSONObject response = executeQuery(String.format(Locale.ROOT, query, TestsConstants.TEST_INDEX_NESTED_TYPE)); - Set allExpectedFieldNames = Stream.of(expectedFieldNames). - flatMap(Set::stream). - collect(toSet()); - - assertContainsColumnsInAnyOrder(getSchema(response), allExpectedFieldNames); - assertContainsData(getDataRows(response), allExpectedFieldNames); - } - - private Set fields(String... fieldNames) { - return Sets.newHashSet(fieldNames); - } - - @Test - public void selectNestedFields() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT nested(message.info), someField FROM %s", - TestsConstants.TEST_INDEX_NESTED_TYPE)); - - List fields = Arrays.asList("message.info", "someField"); - assertContainsColumns(getSchema(response), fields); - assertContainsData(getDataRows(response), fields); - - // The nested test index being used contains 5 entries but one of them has an array of 2 message objects, so - // we check to see if the amount of data rows is 6 since that is the result after flattening - assertThat(getDataRows(response).length(), equalTo(6)); - } - - @Test - public void selectNestedFieldWithWildcard() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT nested(message.*) FROM %s", - TestsConstants.TEST_INDEX_NESTED_TYPE)); - - assertContainsColumnsInAnyOrder(getSchema(response), messageFields); - assertContainsData(getDataRows(response), messageFields); - } - - @Test - public void selectWithWhere() throws IOException { - int balanceToCompare = 30000; - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT balance " + - "FROM %s " + - "WHERE balance > %d", - TestsConstants.TEST_INDEX_ACCOUNT, balanceToCompare)); - - /* - * Previously the DataRows map was used to check specific fields but the JDBC response for "datarows" is a - * JSONArray so keys cannot be used to identify the field. Instead, the expected position will be used to - * retrieve the data from the JSONArray representing each "row". - */ - JSONArray dataRows = getDataRows(response); - for (int i = 0; i < dataRows.length(); i++) { - JSONArray row = dataRows.getJSONArray(i); - int balance = row.getInt(0); - - assertThat(balance, greaterThan(balanceToCompare)); - } - } - - @Test - public void groupBySingleField() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * FROM %s GROUP BY age", - TestsConstants.TEST_INDEX_ACCOUNT)); - - List fields = Collections.singletonList("age"); - assertContainsColumns(getSchema(response), fields); - assertContainsData(getDataRows(response), fields); - } - - @Test - public void groupByMultipleFields() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * FROM %s GROUP BY age, balance", - TestsConstants.TEST_INDEX_ACCOUNT)); - - List fields = Arrays.asList("age", "balance"); - assertContainsColumns(getSchema(response), fields); - assertContainsData(getDataRows(response), fields); - } - - @Test - public void testSizeAndTotal() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * " + - "FROM %s " + - "WHERE balance > 30000 " + - "LIMIT 5", - TestsConstants.TEST_INDEX_ACCOUNT)); - - JSONArray dataRows = getDataRows(response); - assertThat(dataRows.length(), equalTo(5)); - - // The value to compare to here was obtained by running the query in the plugin and looking at the SearchHits - int totalHits = response.getInt("total"); - assertThat(totalHits, equalTo(402)); - } - - @Test - public void testSizeWithGroupBy() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * FROM %s GROUP BY age LIMIT 5", - TestsConstants.TEST_INDEX_ACCOUNT)); - - assertThat(getDataRows(response).length(), equalTo(5)); - } - - @Test - public void aggregationFunctionInSelect() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT COUNT(*) FROM %s GROUP BY age", - TestsConstants.TEST_INDEX_ACCOUNT)); - - List fields = Arrays.asList("COUNT(*)"); - assertContainsColumns(getSchema(response), fields); - - JSONArray dataRows = getDataRows(response); - for (int i = 0; i < dataRows.length(); i++) { - JSONArray row = dataRows.getJSONArray(i); - long countVal = row.getLong(0); - - assertThat(countVal, greaterThan((long) 0)); - } - } - - @Test - public void aggregationFunctionInSelectCaseCheck() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT count(*) FROM %s GROUP BY age", - TestsConstants.TEST_INDEX_ACCOUNT)); - - List fields = Arrays.asList("COUNT(*)"); - assertContainsColumns(getSchema(response), fields); - - JSONArray dataRows = getDataRows(response); - for (int i = 0; i < dataRows.length(); i++) { - JSONArray row = dataRows.getJSONArray(i); - long countVal = row.getLong(0); - - assertThat(countVal, greaterThan((long) 0)); - } - } - - @Test - public void aggregationFunctionInSelectWithAlias() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT COUNT(*) AS total FROM %s GROUP BY age", - TestsConstants.TEST_INDEX_ACCOUNT)); - - List fields = Arrays.asList("total"); - assertContainsColumns(getSchema(response), fields); - - JSONArray dataRows = getDataRows(response); - for (int i = 0; i < dataRows.length(); i++) { - JSONArray row = dataRows.getJSONArray(i); - long countVal = row.getLong(0); - - assertThat(countVal, greaterThan((long) 0)); - } - } - - @Test - public void aggregationFunctionInSelectGroupByMultipleFields() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT SUM(age) FROM %s GROUP BY age, state.keyword", - TestsConstants.TEST_INDEX_ACCOUNT)); - - List fields = Arrays.asList("SUM(age)"); - assertContainsColumns(getSchema(response), fields); - assertContainsData(getDataRows(response), fields); - } - - @Test - public void aggregationFunctionInSelectNoGroupBy() throws IOException { - JSONObject response = executeQuery(String.format(Locale.ROOT, "SELECT SUM(age) FROM %s", - TestsConstants.TEST_INDEX_ACCOUNT)); - - String ageSum = "SUM(age)"; - assertContainsColumns(getSchema(response), Collections.singletonList(ageSum)); - - JSONArray dataRows = getDataRows(response); - for (int i = 0; i < dataRows.length(); i++) { - JSONArray row = dataRows.getJSONArray(i); - double sumVal = row.getDouble(0); - - assertThat(sumVal, greaterThan((double) 0)); - } - } - - @Test - public void multipleAggregationFunctionsInSelect() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT COUNT(*), AVG(age) FROM %s GROUP BY age", - TestsConstants.TEST_INDEX_ACCOUNT)); - - List fields = Arrays.asList("COUNT(*)", "AVG(age)"); - assertContainsColumns(getSchema(response), fields); - assertContainsData(getDataRows(response), fields); - } - - @Test - public void aggregationFunctionInHaving() throws IOException { - JSONObject response = executeQuery(String.format(Locale.ROOT, - "SELECT gender " + - "FROM %s " + - "GROUP BY gender " + - "HAVING count(*) > 500", - TestsConstants.TEST_INDEX_ACCOUNT)); - - String ageSum = "gender"; - assertContainsColumns(getSchema(response), Collections.singletonList(ageSum)); - - JSONArray dataRows = getDataRows(response); - assertEquals(1, dataRows.length()); - assertEquals("m", dataRows.getJSONArray(0).getString(0)); - } - - /** - * This case doesn't seem to be supported by the plugin at the moment. - * Looks like the painless script of the inner function is put inside the aggregation function but - * this syntax may not be correct since it returns 0 which is the default value (since 0 is returned in - * cases like COUNT(wrongField) as well). - */ -// @Test -// public void nestedAggregationFunctionInSelect() { -// String query = String.format(Locale.ROOT, "SELECT SUM(SQRT(age)) FROM age GROUP BY age", TEST_INDEX_ACCOUNT); -// } - - @Test - public void fieldsWithAlias() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT firstname AS first, age AS a FROM %s", - TestsConstants.TEST_INDEX_ACCOUNT)); - - Map aliases = new HashMap<>(); - aliases.put("firstname", "first"); - aliases.put("age", "a"); - - assertContainsAliases(getSchema(response), aliases); - } - - @Test - public void indexWithMissingFields() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT phrase, insert_time2 " + - "FROM %s " + - "WHERE match_phrase(phrase, 'brown fox')", - TestsConstants.TEST_INDEX_PHRASE)); - - JSONArray dataRowEntry = getDataRows(response).getJSONArray(0); - assertThat(dataRowEntry.length(), equalTo(2)); - assertThat(dataRowEntry.get(0), equalTo("brown fox")); - assertThat(dataRowEntry.get(1), equalTo(JSONObject.NULL)); // TODO See if this null check is failing - } - - @Test - public void joinQuery() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT b1.balance, b1.age, b2.firstname " + - "FROM %s b1 JOIN %s b2 ON b1.age = b2.age", - TestsConstants.TEST_INDEX_ACCOUNT, TestsConstants.TEST_INDEX_ACCOUNT)); - - List fields = Arrays.asList("b1.balance", "b1.age", "b2.firstname"); - assertContainsColumns(getSchema(response), fields); - assertContainsData(getDataRows(response), fields); - } - - @Test - public void joinQueryWithAlias() throws IOException { - JSONObject response = executeQuery(String.format(Locale.ROOT, "SELECT b1.balance AS bal, " + - " b1.age AS age, b2.firstname AS name FROM %s b1 JOIN %s b2 ON b1.age = b2.age", - TestsConstants.TEST_INDEX_ACCOUNT, TestsConstants.TEST_INDEX_ACCOUNT)); - - Map aliases = new HashMap<>(); - aliases.put("b1.balance", "bal"); - aliases.put("b1.age", "age"); - aliases.put("b2.firstname", "name"); - - assertContainsAliases(getSchema(response), aliases); - assertContainsData(getDataRows(response), Arrays.asList("bal", "age", "name")); - } - - @Test - public void joinQueryWithObjectFieldInSelect() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT c.name.firstname, d.name.lastname " + - "FROM %s c JOIN %s d ON d.hname = c.house", - TestsConstants.TEST_INDEX_GAME_OF_THRONES, - TestsConstants.TEST_INDEX_GAME_OF_THRONES)); - - List fields = Arrays.asList("c.name.firstname", "d.name.lastname"); - assertContainsColumns(getSchema(response), fields); - - // d.name.lastname is null here since entries with hname don't have a name.lastname entry, so only length is - // checked - JSONArray dataRows = getDataRows(response); - assertThat(dataRows.length(), greaterThan(0)); - - JSONArray row = dataRows.getJSONArray(0); - assertThat(row.length(), equalTo(fields.size())); - } - - @Test - public void joinQuerySelectOnlyOnOneTable() throws Exception { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT b1.age " + - "FROM %s b1 JOIN %s b2 ON b1.firstname = b2.firstname", - TestsConstants.TEST_INDEX_ACCOUNT, TestsConstants.TEST_INDEX_ACCOUNT)); - - List fields = Collections.singletonList("b1.age"); - assertContainsColumns(getSchema(response), fields); - assertContainsData(getDataRows(response), fields); - } - - @Test - public void fieldOrder() throws IOException { - - final String[] expectedFields = {"age", "firstname", "address", "gender", "email"}; - final Object[] expectedValues = {32, "Amber", "880 Holmes Lane", "M", "amberduke@pyrami.com"}; - - testFieldOrder(expectedFields, expectedValues); - } - - @Test - public void fieldOrderOther() throws IOException { - - final String[] expectedFields = {"email", "firstname", "age", "gender", "address"}; - final Object[] expectedValues = {"amberduke@pyrami.com", "Amber", 32, "M", "880 Holmes Lane"}; - - testFieldOrder(expectedFields, expectedValues); - } - - private void testFieldOrder(final String[] expectedFields, final Object[] expectedValues) throws IOException { - - final String fields = String.join(", ", expectedFields); - final String query = String.format(Locale.ROOT, "SELECT %s FROM %s " + - "WHERE email='amberduke@pyrami.com'", fields, TestsConstants.TEST_INDEX_ACCOUNT); - final JSONObject result = executeQuery(query); - - for (int i = 0; i < expectedFields.length; ++i) { - - final String fieldName = (String)result.query(String.format(Locale.ROOT, "/schema/%d/name", i)); - assertThat(fieldName, equalTo(expectedFields[i])); - final Object fieldValue = result.query(String.format(Locale.ROOT, "/datarows/0/%d", i)); - assertThat(fieldValue, equalTo(expectedValues[i])); - } - } - - private JSONArray getSchema(JSONObject jdbcResponse) { return jdbcResponse.getJSONArray("schema"); } - - private JSONArray getDataRows(JSONObject jdbcResponse) { - return jdbcResponse.getJSONArray("datarows"); - } - - private void assertContainsColumnsInAnyOrder(JSONArray schema, Set fields) { - - assertThat(schema.length(), equalTo(fields.size())); - - for (int i = 0; i < schema.length(); i++) { - JSONObject column = schema.getJSONObject(i); - String name = column.getString("name"); - - assertTrue(fields.contains(name)); - } - } - - private void assertContainsColumns(JSONArray schema, List fields) { - - assertThat(schema.length(), equalTo(fields.size())); - - for (int i = 0; i < schema.length(); i++) { - JSONObject column = schema.getJSONObject(i); - String name = column.getString("name"); - - assertThat(name, equalTo(fields.get(i))); - } - } - - private void assertContainsAliases(JSONArray schema, Map aliases) { - for (int i = 0; i < schema.length(); i++) { - JSONObject column = schema.getJSONObject(i); - assertTrue(column.has("alias")); - - String name = column.getString("name"); - String alias = column.getString("alias"); - - assertThat(alias, equalTo(aliases.get(name))); - } - } - - private void assertContainsData(JSONArray dataRows, Collection fields) { - assertThat(dataRows.length(), greaterThan(0)); - JSONArray row = dataRows.getJSONArray(0); - - assertThat(row.length(), equalTo(fields.size())); - for (int i = 0; i < row.length(); i++) { - assertThat(row.get(i), not(equalTo(JSONObject.NULL))); - } - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/PrettyFormatterIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/PrettyFormatterIT.java deleted file mode 100644 index 83e0f17a2f..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/PrettyFormatterIT.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; -import com.google.common.io.Files; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; - -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_ACCOUNT; -import static org.hamcrest.Matchers.equalTo; - -public class PrettyFormatterIT extends SQLIntegTestCase{ - - @Override - protected void init() throws Exception { - loadIndex(Index.ACCOUNT); - } - - @Test - public void assertExplainPrettyFormatted() throws IOException { - String query = StringUtils.format("SELECT firstname FROM %s", TEST_INDEX_ACCOUNT); - - String notPrettyExplainOutputFilePath = TestUtils.getResourceFilePath( - "src/test/resources/expectedOutput/explainIT_format_not_pretty.json"); - String notPrettyExplainOutput = Files.toString(new File(notPrettyExplainOutputFilePath), StandardCharsets.UTF_8); - - assertThat(executeExplainRequest(query, ""), equalTo(notPrettyExplainOutput)); - assertThat(executeExplainRequest(query, "pretty=false"), equalTo(notPrettyExplainOutput)); - - String prettyExplainOutputFilePath = TestUtils.getResourceFilePath( - "src/test/resources/expectedOutput/explainIT_format_pretty.json"); - String prettyExplainOutput = Files.toString(new File(prettyExplainOutputFilePath), StandardCharsets.UTF_8); - - assertThat(executeExplainRequest(query, "pretty"), equalTo(prettyExplainOutput)); - assertThat(executeExplainRequest(query, "pretty=true"), equalTo(prettyExplainOutput)); - } - - private String executeExplainRequest(String query, String explainParam) throws IOException { - String endpoint = "/_opendistro/_sql/_explain?" + explainParam; - String request = makeRequest(query); - - Request sqlRequest = new Request("POST", endpoint); - sqlRequest.setJsonEntity(request); - - Response response = client().performRequest(sqlRequest); - String responseString = TestUtils.getResponseBody(response, true); - - return responseString; - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/QueryAnalysisIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/QueryAnalysisIT.java deleted file mode 100644 index 6f124dce72..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/QueryAnalysisIT.java +++ /dev/null @@ -1,351 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import com.amazon.opendistroforelasticsearch.sql.antlr.semantic.SemanticAnalysisException; -import com.amazon.opendistroforelasticsearch.sql.antlr.syntax.SyntaxAnalysisException; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlFeatureNotImplementedException; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; -import org.elasticsearch.rest.RestStatus; -import org.junit.Assert; -import org.junit.Test; - -import java.io.IOException; - -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.QUERY_ANALYSIS_ENABLED; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_SUGGESTION; -import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_THRESHOLD; -import static org.elasticsearch.rest.RestStatus.BAD_REQUEST; -import static org.elasticsearch.rest.RestStatus.OK; -import static org.elasticsearch.rest.RestStatus.SERVICE_UNAVAILABLE; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; - -/** - * Integration test for syntax and semantic analysis against query by new ANTLR parser. - */ -public class QueryAnalysisIT extends SQLIntegTestCase { - - @Override - protected void init() throws Exception { - loadIndex(Index.BANK); - } - - @Test - public void missingFromClauseShouldThrowSyntaxException() { - queryShouldThrowSyntaxException("SELECT 1"); - } - - @Test - public void unsupportedOperatorShouldThrowSyntaxException() { - queryShouldThrowSyntaxException( - "SELECT * FROM elasticsearch-sql_test_index_bank WHERE age <=> 1" - ); - } - - @Test - public void unsupportedOperatorShouldSkipAnalysisAndThrowOtherExceptionIfAnalyzerDisabled() { - runWithClusterSetting( - new ClusterSetting("transient", QUERY_ANALYSIS_ENABLED, "false"), - () -> queryShouldThrowException( - "SELECT * FROM elasticsearch-sql_test_index_bank WHERE age <=> 1", - SqlParseException.class - ) - ); - } - - @Test - public void suggestionForWrongFieldNameShouldBeProvidedIfSuggestionEnabled() { - runWithClusterSetting( - new ClusterSetting("transient", QUERY_ANALYSIS_SEMANTIC_SUGGESTION, "true"), - () -> queryShouldThrowSemanticException( - "SELECT * FROM elasticsearch-sql_test_index_bank b WHERE a.balance = 1000", - "Field [a.balance] cannot be found or used here.", - "Did you mean [b.balance]?" - ) - ); - } - - @Test - public void wrongFieldNameShouldPassIfIndexMappingIsVeryLarge() { - runWithClusterSetting( - new ClusterSetting("transient", QUERY_ANALYSIS_SEMANTIC_THRESHOLD, "5"), - () -> queryShouldPassAnalysis("SELECT * FROM elasticsearch-sql_test_index_bank WHERE age123 = 1") - ); - } - - /* - @Test - public void useNewAddedFieldShouldPass() throws Exception { - // 1.Make sure new add fields not there originally - String query = "SELECT salary FROM elasticsearch-sql_test_index_bank WHERE education = 'PhD'"; - queryShouldThrowSemanticException(query, "Field [education] cannot be found or used here."); - - // 2.Index an document with fields not present in mapping previously - String docWithNewFields = "{\"account_number\":12345,\"education\":\"PhD\",\"salary\": \"10000\"}"; - IndexResponse resp = client().index(new IndexRequest().index("elasticsearch-sql_test_index_bank"). - source(docWithNewFields, JSON)).get(); - - Assert.assertEquals(RestStatus.CREATED, resp.status()); - - // 3.Same query should pass - executeQuery(query); - } - */ - - @Test - public void nonExistingFieldNameShouldThrowSemanticException() { - queryShouldThrowSemanticException( - "SELECT * FROM elasticsearch-sql_test_index_bank WHERE balance1 = 1000", - "Field [balance1] cannot be found or used here." - //"Did you mean [balance]?" - ); - } - - @Test - public void nonExistingIndexAliasShouldThrowSemanticException() { - queryShouldThrowSemanticException( - "SELECT * FROM elasticsearch-sql_test_index_bank b WHERE a.balance = 1000", - "Field [a.balance] cannot be found or used here." - //"Did you mean [b.balance]?" - ); - } - - @Test - public void indexJoinNonNestedFieldShouldThrowSemanticException() { - queryShouldThrowSemanticException( - "SELECT * FROM elasticsearch-sql_test_index_bank b1, b1.firstname f1", - "Operator [JOIN] cannot work with [INDEX, TEXT]." - ); - } - - @Test - public void scalarFunctionCallWithTypoInNameShouldThrowSemanticException() { - queryShouldThrowSemanticException( - "SELECT * FROM elasticsearch-sql_test_index_bank WHERE ABSa(age) = 1", - "Function [ABSA] cannot be found or used here.", - "Did you mean [ABS]?" - ); - } - - @Test - public void scalarFunctionCallWithWrongTypeArgumentShouldThrowSemanticException() { - queryShouldThrowSemanticException( - "SELECT * FROM elasticsearch-sql_test_index_bank WHERE LOG(lastname) = 1", - "Function [LOG] cannot work with [KEYWORD].", - "Usage: LOG(NUMBER T) -> DOUBLE or LOG(NUMBER T, NUMBER) -> DOUBLE" - ); - } - - @Test - public void aggregateFunctionCallWithWrongNumberOfArgumentShouldThrowSemanticException() { - queryShouldThrowSemanticException( - "SELECT city FROM elasticsearch-sql_test_index_bank GROUP BY city HAVING MAX(age, birthdate) > 1", - "Function [MAX] cannot work with [INTEGER, DATE].", - "Usage: MAX(NUMBER T) -> T" - ); - } - - @Test - public void compareIntegerFieldWithBooleanShouldThrowSemanticException() { - queryShouldThrowSemanticException( - "SELECT * FROM elasticsearch-sql_test_index_bank b WHERE b.age IS FALSE", - "Operator [IS] cannot work with [INTEGER, BOOLEAN].", - "Usage: Please use compatible types from each side." - ); - } - - @Test - public void compareNumberFieldWithStringShouldThrowSemanticException() { - queryShouldThrowSemanticException( - "SELECT * FROM elasticsearch-sql_test_index_bank b WHERE b.age >= 'test'", - "Operator [>=] cannot work with [INTEGER, STRING].", - "Usage: Please use compatible types from each side." - ); - } - - @Test - public void compareLogFunctionCallWithNumberFieldWithStringShouldThrowSemanticException() { - queryShouldThrowSemanticException( - "SELECT * FROM elasticsearch-sql_test_index_bank b WHERE LOG(b.balance) != 'test'", - "Operator [!=] cannot work with [DOUBLE, STRING].", - "Usage: Please use compatible types from each side." - ); - } - - @Test - public void unionNumberFieldWithStringShouldThrowSemanticException() { - queryShouldThrowSemanticException( - "SELECT age FROM elasticsearch-sql_test_index_bank" + - " UNION SELECT address FROM elasticsearch-sql_test_index_bank", - "Operator [UNION] cannot work with [INTEGER, TEXT]." - ); - } - - @Test - public void minusBooleanFieldWithDateShouldThrowSemanticException() { - queryShouldThrowSemanticException( - "SELECT male FROM elasticsearch-sql_test_index_bank" + - " MINUS SELECT birthdate FROM elasticsearch-sql_test_index_bank", - "Operator [MINUS] cannot work with [BOOLEAN, DATE]." - ); - } - - @Test - public void useInClauseWithIncompatibleFieldTypesShouldFail() { - queryShouldThrowSemanticException( - "SELECT * FROM elasticsearch-sql_test_index_bank WHERE male " + - " IN (SELECT 1 FROM elasticsearch-sql_test_index_bank)", - "Operator [IN] cannot work with [BOOLEAN, INTEGER]." - ); - } - - @Test - public void queryWithNestedFunctionShouldFail() { - queryShouldThrowFeatureNotImplementedException( - "SELECT abs(log(balance)) FROM elasticsearch-sql_test_index_bank", - "Nested function calls like [abs(log(balance))] are not supported yet" - ); - } - - @Test - public void nestedFunctionWithMathConstantAsInnerFunctionShouldPass() { - queryShouldPassAnalysis("SELECT log(e()) FROM elasticsearch-sql_test_index_bank"); - } - - @Test - public void aggregateWithFunctionAggregatorShouldFail() { - queryShouldThrowFeatureNotImplementedException( - "SELECT max(log(age)) FROM elasticsearch-sql_test_index_bank", - "Aggregation calls with function aggregator like [max(log(age))] are not supported yet" - ); - } - - @Test - public void queryWithUnsupportedFunctionShouldFail() { - queryShouldThrowFeatureNotImplementedException( - "SELECT balance DIV age FROM elasticsearch-sql_test_index_bank", - "Operator [DIV] is not supported yet" - ); - } - - @Test - public void useNegativeNumberConstantShouldPass() { - queryShouldPassAnalysis( - "SELECT * FROM elasticsearch-sql_test_index_bank " + - "WHERE age > -1 AND balance < -123.456789" - ); - } - - /** Run the query with cluster setting changed and cleaned after complete */ - private void runWithClusterSetting(ClusterSetting setting, Runnable query) { - try { - updateClusterSettings(setting); - query.run(); - } - catch (IOException e) { - throw new IllegalStateException( - StringUtils.format("Exception raised when running with cluster setting [%s]", setting)); - } - finally { - // Clean up or ES will throw java.lang.AssertionError: test leaves persistent cluster metadata behind - try { - updateClusterSettings(setting.nullify()); - } - catch (IOException e) { - // Ignore exception during the cleanup - } - } - } - - private void queryShouldThrowSyntaxException(String query, String... expectedMsgs) { - queryShouldThrowException(query, SyntaxAnalysisException.class, expectedMsgs); - } - - private void queryShouldThrowSemanticException(String query, String... expectedMsgs) { - queryShouldThrowException(query, SemanticAnalysisException.class, expectedMsgs); - } - - private void queryShouldThrowFeatureNotImplementedException(String query, String... expectedMsgs) { - queryShouldThrowExceptionWithRestStatus(query, SqlFeatureNotImplementedException.class, SERVICE_UNAVAILABLE, expectedMsgs); - } - - private void queryShouldThrowException(String query, Class exceptionType, String... expectedMsgs) { - queryShouldThrowExceptionWithRestStatus(query, exceptionType, BAD_REQUEST, expectedMsgs); - } - - private void queryShouldThrowExceptionWithRestStatus(String query, Class exceptionType, RestStatus status, String... expectedMsgs) { - try { - executeQuery(query); - Assert.fail("Expected ResponseException, but none was thrown for query: " + query); - } - catch (ResponseException e) { - ResponseAssertion assertion = new ResponseAssertion(e.getResponse()); - assertion.assertStatusEqualTo(status.getStatus()); - assertion.assertBodyContains("\"type\": \"" + exceptionType.getSimpleName() + "\""); - for (String msg : expectedMsgs) { - assertion.assertBodyContains(msg); - } - } - catch (IOException e) { - throw new IllegalStateException( - "Unexpected IOException raised rather than expected AnalysisException for query: " + query); - } - } - - private void queryShouldPassAnalysis(String query) { - String endpoint = "/_opendistro/_sql?"; - String requestBody = makeRequest(query); - Request sqlRequest = new Request("POST", endpoint); - sqlRequest.setJsonEntity(requestBody); - - try { - Response response = client().performRequest(sqlRequest); - ResponseAssertion assertion = new ResponseAssertion(response); - assertion.assertStatusEqualTo(OK.getStatus()); - } catch (IOException e) { - throw new IllegalStateException("Unexpected IOException raised for query: " + query); - } - } - - private static class ResponseAssertion { - private final Response response; - private final String body; - - ResponseAssertion(Response response) { - this.response = response; - try { - this.body = TestUtils.getResponseBody(response); - } - catch (IOException e) { - throw new IllegalStateException("Unexpected IOException raised when reading response body"); - } - } - - void assertStatusEqualTo(int expectedStatus) { - assertThat(response.getStatusLine().getStatusCode(), equalTo(expectedStatus)); - } - - void assertBodyContains(String content) { - assertThat(body, containsString(content)); - } - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/QueryFunctionsIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/QueryFunctionsIT.java deleted file mode 100644 index 2b87ce75b6..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/QueryFunctionsIT.java +++ /dev/null @@ -1,295 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.search.SearchHit; -import org.hamcrest.BaseMatcher; -import org.hamcrest.Description; -import org.hamcrest.FeatureMatcher; -import org.hamcrest.Matcher; -import org.json.JSONObject; -import org.junit.Test; - -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_ACCOUNT; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_NESTED_TYPE; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_PHRASE; -import static org.hamcrest.Matchers.anyOf; -import static org.hamcrest.Matchers.everyItem; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.startsWith; - -public class QueryFunctionsIT extends SQLIntegTestCase { - - private static final String SELECT_ALL = "SELECT *"; - private static final String FROM_ACCOUNTS = "FROM " + TEST_INDEX_ACCOUNT; - private static final String FROM_NESTED = "FROM " + TEST_INDEX_NESTED_TYPE; - private static final String FROM_PHRASE = "FROM " + TEST_INDEX_PHRASE; - - /** - * TODO Looks like Math/Date Functions test all use the same query() and execute() functions - * TODO execute/featureValueOf/hits functions are the same as used in NestedFieldQueryIT, should refactor into util - */ - - @Override - protected void init() throws Exception { - loadIndex(Index.ACCOUNT); - loadIndex(Index.NESTED); - loadIndex(Index.PHRASE); - } - - @Test - public void query() throws IOException { - assertThat( - query( - "SELECT state", - FROM_ACCOUNTS, - "WHERE QUERY('CA')" - ), - hits( - hasValueForFields("CA", "state") - ) - ); - } - - @Test - public void matchQueryRegularField() throws IOException { - assertThat( - query( - "SELECT firstname", - FROM_ACCOUNTS, - "WHERE MATCH_QUERY(firstname, 'Ayers')" - ), - hits( - hasValueForFields("Ayers", "firstname") - ) - ); - } - - @Test - public void matchQueryNestedField() throws IOException { - SearchHit[] hits = query("SELECT comment.data", FROM_NESTED, "WHERE MATCH_QUERY(NESTED(comment.data), 'aa')").getHits().getHits(); - Map source = hits[0].getSourceAsMap(); - // SearchHits innerHits = hits[0].getInnerHits().get("comment"); - assertThat( - query( - "SELECT comment.data", - FROM_NESTED, - "WHERE MATCH_QUERY(NESTED(comment.data), 'aa')" - ), - hits( - anyOf(hasNestedField("comment", "data", "aa"), - hasNestedArrayField("comment", "data", "aa")) - ) - ); - } - - @Test - public void scoreQuery() throws IOException { - assertThat( - query( - "SELECT firstname", - FROM_ACCOUNTS, - "WHERE SCORE(MATCH_QUERY(firstname, 'Ayers'), 10)" - ), - hits( - hasValueForFields("Ayers", "firstname") - ) - ); - } - - @Test - public void scoreQueryWithNestedField() throws IOException { - assertThat( - query( - "SELECT comment.data", - FROM_NESTED, - "WHERE SCORE(MATCH_QUERY(NESTED(comment.data), 'ab'), 10)" - ), - hits( - //hasValueForFields("ab", "comment.data") - hasNestedField("comment", - "data", "ab") - ) - ); - } - - @Test - public void wildcardQuery() throws IOException { - assertThat( - query( - "SELECT city", - FROM_ACCOUNTS, - "WHERE WILDCARD_QUERY(city.keyword, 'B*')" - ), - hits( - hasFieldWithPrefix("city", "B") - ) - ); - } - - @Test - public void matchPhraseQuery() throws IOException { - assertThat( - query( - "SELECT phrase", - FROM_PHRASE, - "WHERE MATCH_PHRASE(phrase, 'brown fox')" - ), - hits( - hasValueForFields("brown fox", "phrase") - ) - ); - } - - @Test - public void multiMatchQuerySingleField() throws IOException { - assertThat( - query( - "SELECT firstname", - FROM_ACCOUNTS, - "WHERE MULTI_MATCH('query'='Ayers', 'fields'='firstname')" - ), - hits( - hasValueForFields("Ayers", "firstname") - ) - ); - } - - @Test - public void multiMatchQueryWildcardField() throws IOException { - assertThat( - query( - "SELECT firstname, lastname", - FROM_ACCOUNTS, - "WHERE MULTI_MATCH('query'='Bradshaw', 'fields'='*name')" - ), - hits( - hasValueForFields("Bradshaw", "firstname", "lastname") - ) - ); - } - - @Test - public void numberLiteralInSelectField() { - assertTrue( - executeQuery(StringUtils.format("SELECT 234234 AS number from %s", TEST_INDEX_ACCOUNT), "jdbc") - .contains("234234") - ); - - assertTrue( - executeQuery(StringUtils.format("SELECT 2.34234 AS number FROM %s", TEST_INDEX_ACCOUNT), "jdbc") - .contains("2.34234") - ); - } - - private final Matcher hits(Matcher subMatcher) { - return featureValueOf("hits", everyItem(subMatcher), resp -> Arrays.asList(resp.getHits().getHits())); - } - - private FeatureMatcher featureValueOf(String name, Matcher subMatcher, Function getter) { - return new FeatureMatcher(subMatcher, name, name) { - @Override - protected U featureValueOf(T actual) { - return getter.apply(actual); - } - }; - } - - /** - * Create Matchers for each field -> value - * Only one of the Matchers need to match (per hit) - * - * Ex. If a query with wildcard field is made: - * multi_match(query="Ayers", fields="*name") - * - * Then the value "Ayers" can be found in either the firstname or lastname field. Only one of these fields - * need to satisfy the query value to be evaluated as correct expected output. - * - * @param value The value to match for a field in the sourceMap - * @param fields A list of fields to match - */ - @SafeVarargs - private final Matcher hasValueForFields(String value, String... fields) { - return anyOf( - Arrays.asList(fields). - stream(). - map(field -> kv(field, is(value))). - collect(Collectors.toList())); - } - - private final Matcher hasFieldWithPrefix(String field, String prefix) { - return featureValueOf(field, startsWith(prefix), hit -> (String) hit.getSourceAsMap().get(field)); - } - - private final Matcher hasNestedField(String path, String field, String value) { - return featureValueOf(field, is(value), hit -> ((HashMap) hit.getSourceAsMap().get(path)).get(field)); - } - - private final Matcher hasNestedArrayField(String path, String field, String value) { - - return new BaseMatcher() { - @Override - public void describeTo(Description description) { - - } - - @Override - public boolean matches(Object item) { - - SearchHit hit = (SearchHit)item; - List elements = (List) ((HashMap) hit.getSourceAsMap().get(path)).get(field); - return elements.contains(value); - } - }; - } - - private Matcher kv(String key, Matcher valMatcher) { - return featureValueOf(key, valMatcher, hit -> hit.getSourceAsMap().get(key)); - } - - /*********************************************************** - Query Utility to Fetch Response for SQL - ***********************************************************/ - - private SearchResponse query(String select, String from, String... statements) throws IOException { - return execute(select + " " + from + " " + String.join(" ", statements)); - } - - private SearchResponse execute(String sql) throws IOException { - final JSONObject jsonObject = executeQuery(sql); - - final XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser( - NamedXContentRegistry.EMPTY, - LoggingDeprecationHandler.INSTANCE, - jsonObject.toString()); - return SearchResponse.fromXContent(parser); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/QueryIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/QueryIT.java deleted file mode 100644 index b3d7adfd87..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/QueryIT.java +++ /dev/null @@ -1,1791 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; -import org.elasticsearch.client.ResponseException; -import org.elasticsearch.rest.RestStatus; -import org.joda.time.DateTime; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; -import org.json.JSONArray; -import org.json.JSONObject; -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.Locale; -import java.util.Set; - -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_ACCOUNT; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_BANK; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_GAME_OF_THRONES; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_NESTED_TYPE; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_ONLINE; -import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.isOneOf; -import static org.hamcrest.Matchers.lessThan; -import static org.hamcrest.Matchers.lessThanOrEqualTo; -import static org.hamcrest.Matchers.not; - -public class QueryIT extends SQLIntegTestCase { - - /** - * Currently commenting out tests related to JoinType index since there is an issue with mapping. - * - * Also ignoring the following tests as they are failing, will require investigation: - * - idsQuerySubQueryIds - * - escapedCharactersCheck - * - fieldCollapsingTest - * - idsQueryOneId - * - idsQueryMultipleId - * - multipleIndicesOneNotExistWithoutHint - * - * The following tests are being ignored because subquery is still running in ES transport thread: - * - twoSubQueriesTest() - * - inTermsSubQueryTest() - */ - - final static int BANK_INDEX_MALE_TRUE = 4; - final static int BANK_INDEX_MALE_FALSE = 3; - - @Override - protected void init() throws Exception { - loadIndex(Index.ONLINE); - loadIndex(Index.ACCOUNT); - loadIndex(Index.PHRASE); - loadIndex(Index.DOG); - loadIndex(Index.PEOPLE); - loadIndex(Index.GAME_OF_THRONES); - loadIndex(Index.ODBC); - loadIndex(Index.LOCATION); - loadIndex(Index.NESTED); - // TODO Remove comment after issue with loading join type is resolved - // loadIndex(Index.JOIN); - loadIndex(Index.BANK); - loadIndex(Index.BANK_TWO); - loadIndex(Index.BANK_WITH_NULL_VALUES); - } - - @Test - public void searchTypeTest() throws IOException { - JSONObject response = executeQuery(String.format(Locale.ROOT, "SELECT * FROM %s LIMIT 1000", - TestsConstants.TEST_INDEX_PHRASE)); - Assert.assertTrue(response.has("hits")); - Assert.assertEquals(6, getTotalHits(response)); - } - - @Test - public void multipleFromTest() throws IOException { - JSONObject response = executeQuery(String.format(Locale.ROOT, - "SELECT * FROM %s, %s LIMIT 2000", - TestsConstants.TEST_INDEX_BANK, TestsConstants.TEST_INDEX_BANK_TWO)); - Assert.assertTrue(response.has("hits")); - Assert.assertEquals(14, getTotalHits(response)); - } - - @Test - public void selectAllWithFieldReturnsAll() throws IOException { - JSONObject response = executeQuery(StringUtils.format( - "SELECT *, age " + - "FROM %s " + - "LIMIT 5", - TestsConstants.TEST_INDEX_BANK - )); - - checkSelectAllAndFieldResponseSize(response); - } - - @Test - public void selectAllWithFieldReverseOrder() throws IOException { - JSONObject response = executeQuery(StringUtils.format( - "SELECT *, age " + - "FROM %s " + - "LIMIT 5", - TestsConstants.TEST_INDEX_BANK - )); - - checkSelectAllAndFieldResponseSize(response); - } - - @Test - public void selectAllWithMultipleFields() throws IOException { - JSONObject response = executeQuery(StringUtils.format( - "SELECT *, age, address " + - "FROM %s " + - "LIMIT 5", - TestsConstants.TEST_INDEX_BANK - )); - - checkSelectAllAndFieldResponseSize(response); - } - - @Test - public void selectAllWithFieldAndOrderBy() throws IOException { - JSONObject response = executeQuery(StringUtils.format( - "SELECT *, age " + - "FROM %s " + - "ORDER BY age " + - "LIMIT 5", - TestsConstants.TEST_INDEX_BANK - )); - - checkSelectAllAndFieldResponseSize(response); - } - - @Test - public void selectAllWithFieldAndGroupBy() throws IOException { - JSONObject response = executeQuery(StringUtils.format( - "SELECT *, age " + - "FROM %s " + - "GROUP BY age " + - "LIMIT 10", - TestsConstants.TEST_INDEX_BANK - )); - - checkSelectAllAndFieldAggregationResponseSize(response, "age"); - } - - @Test - public void selectAllWithFieldAndGroupByReverseOrder() throws IOException { - JSONObject response = executeQuery(StringUtils.format( - "SELECT *, age " + - "FROM %s " + - "GROUP BY age " + - "LIMIT 10", - TestsConstants.TEST_INDEX_BANK - )); - - checkSelectAllAndFieldAggregationResponseSize(response, "age"); - } - - @Test - public void selectFieldWithAliasAndGroupBy() { - String response = executeQuery("SELECT lastname AS name FROM " + TEST_INDEX_ACCOUNT + " GROUP BY name", - "jdbc"); - assertThat(response, containsString("\"alias\": \"name\"")); - } - - public void indexWithWildcardTest() throws IOException { - JSONObject response = executeQuery(String.format(Locale.ROOT, "SELECT * FROM %s* LIMIT 1000", - TestsConstants.TEST_INDEX_BANK)); - Assert.assertTrue(response.has("hits")); - assertThat(getTotalHits(response), greaterThan(0)); - } - - @Test - public void selectSpecificFields() throws IOException { - String[] arr = new String[]{"age", "account_number"}; - Set expectedSource = new HashSet<>(Arrays.asList(arr)); - - JSONObject response = executeQuery(String.format(Locale.ROOT, "SELECT age, account_number FROM %s", - TEST_INDEX_ACCOUNT)); - assertResponseForSelectSpecificFields(response, expectedSource); - } - - @Test - public void selectSpecificFieldsUsingTableAlias() throws IOException { - String[] arr = new String[]{"age", "account_number"}; - Set expectedSource = new HashSet<>(Arrays.asList(arr)); - - JSONObject response = executeQuery(String.format(Locale.ROOT, "SELECT a.age, a.account_number FROM %s a", - TEST_INDEX_ACCOUNT)); - assertResponseForSelectSpecificFields(response, expectedSource); - } - - @Test - public void selectSpecificFieldsUsingTableNamePrefix() throws IOException { - String[] arr = new String[]{"age", "account_number"}; - Set expectedSource = new HashSet<>(Arrays.asList(arr)); - - JSONObject response = executeQuery(String.format(Locale.ROOT, - "SELECT elasticsearch-sql_test_index_account.age, elasticsearch-sql_test_index_account.account_number" + - " FROM %s", - TEST_INDEX_ACCOUNT)); - assertResponseForSelectSpecificFields(response, expectedSource); - } - - private void assertResponseForSelectSpecificFields(JSONObject response, Set expectedSource) { - JSONArray hits = getHits(response); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - Assert.assertEquals(expectedSource, getSource(hit).keySet()); - } - } - - @Ignore("Will fix this in issue https://github.com/opendistro-for-elasticsearch/sql/issues/121") - @Test - public void selectFieldWithSpace() throws IOException { - String[] arr = new String[] {"test field"}; - Set expectedSource = new HashSet<>(Arrays.asList(arr)); - - JSONObject response = executeQuery(String.format(Locale.ROOT, "SELECT ['test field'] FROM %s " + - "WHERE ['test field'] IS NOT null", - TestsConstants.TEST_INDEX_PHRASE)); - - JSONArray hits = getHits(response); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - Assert.assertEquals(expectedSource, getSource(hit).keySet()); - } - } - - @Ignore("field aliases are not supported currently") - // it might be possible to change field names after the query already executed. - @Test - public void selectAliases() throws IOException { - - String[] arr = new String[] {"myage", "myaccount_number"}; - Set expectedSource = new HashSet<>(Arrays.asList(arr)); - - JSONObject result = executeQuery(String.format(Locale.ROOT, - "SELECT age AS myage, account_number AS myaccount_number FROM %s", TEST_INDEX_ACCOUNT)); - JSONArray hits = getHits(result); - hits.forEach(hitObj -> { - JSONObject hit = (JSONObject)hitObj; - Assert.assertEquals(expectedSource, hit.getJSONObject("_source").keySet()); - }); - } - - @Test - public void useTableAliasInWhereClauseTest() throws IOException { - JSONObject response = executeQuery(String.format(Locale.ROOT, - "SELECT * FROM %s a WHERE a.city = 'Nogal' LIMIT 1000", TEST_INDEX_ACCOUNT)); - - JSONArray hits = getHits(response); - Assert.assertEquals(1, getTotalHits(response)); - Assert.assertEquals("Nogal", getSource(hits.getJSONObject(0)).get("city")); - } - - @Test - public void notUseTableAliasInWhereClauseTest() throws IOException { - JSONObject response = executeQuery(String.format(Locale.ROOT, - "SELECT * FROM %s a WHERE city = 'Nogal' LIMIT 1000", TEST_INDEX_ACCOUNT)); - - JSONArray hits = getHits(response); - Assert.assertEquals(1, getTotalHits(response)); - Assert.assertEquals("Nogal", getSource(hits.getJSONObject(0)).get("city")); - } - - @Test - public void useTableNamePrefixInWhereClauseTest() throws IOException { - JSONObject response = executeQuery(String.format(Locale.ROOT, - "SELECT * FROM %s WHERE elasticsearch-sql_test_index_account.city = 'Nogal' LIMIT 1000", - TEST_INDEX_ACCOUNT - )); - - JSONArray hits = getHits(response); - Assert.assertEquals(1, getTotalHits(response)); - Assert.assertEquals("Nogal", getSource(hits.getJSONObject(0)).get("city")); - } - - @Test - public void equalityTest() throws IOException { - JSONObject response = executeQuery(String.format(Locale.ROOT, - "SELECT * FROM %s WHERE city = 'Nogal' LIMIT 1000", TEST_INDEX_ACCOUNT)); - - JSONArray hits = getHits(response); - Assert.assertEquals(1, getTotalHits(response)); - Assert.assertEquals("Nogal", getSource(hits.getJSONObject(0)).get("city")); - } - - @Test - public void equalityTestPhrase() throws IOException { - JSONObject response = executeQuery(String.format(Locale.ROOT, "SELECT * FROM %s WHERE " + - "match_phrase(phrase, 'quick fox here') LIMIT 1000", - TestsConstants.TEST_INDEX_PHRASE)); - - JSONArray hits = getHits(response); - Assert.assertEquals(1, getTotalHits(response)); - Assert.assertEquals("quick fox here", getSource(hits.getJSONObject(0)).getString("phrase")); - } - - @Test - public void greaterThanTest() throws IOException { - int someAge = 25; - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * FROM %s WHERE age > %s LIMIT 1000", - TestsConstants.TEST_INDEX_PEOPLE, - someAge)); - - JSONArray hits = getHits(response); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - int age = getSource(hit).getInt("age"); - assertThat(age, greaterThan(someAge)); - } - } - - @Test - public void greaterThanOrEqualTest() throws IOException { - int someAge = 25; - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * FROM %s WHERE age >= %s LIMIT 1000", - TEST_INDEX_ACCOUNT, - someAge)); - - boolean isEqualFound = false; - JSONArray hits = getHits(response); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - int age = getSource(hit).getInt("age"); - assertThat(age, greaterThanOrEqualTo(someAge)); - - if (age == someAge) - isEqualFound = true; - } - - Assert.assertTrue( - String.format(Locale.ROOT, "At least one of the documents need to contains age equal to %s", someAge), - isEqualFound); - } - - @Test - public void lessThanTest() throws IOException { - int someAge = 25; - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * FROM %s WHERE age < %s LIMIT 1000", - TestsConstants.TEST_INDEX_PEOPLE, - someAge)); - - JSONArray hits = getHits(response); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - int age = getSource(hit).getInt("age"); - assertThat(age, lessThan(someAge)); - } - } - - @Test - public void lessThanOrEqualTest() throws IOException { - int someAge = 25; - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * FROM %s WHERE age <= %s LIMIT 1000", - TEST_INDEX_ACCOUNT, - someAge)); - - boolean isEqualFound = false; - JSONArray hits = getHits(response); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - int age = getSource(hit).getInt("age"); - assertThat(age, lessThanOrEqualTo(someAge)); - - if (age == someAge) - isEqualFound = true; - } - - Assert.assertTrue( - String.format(Locale.ROOT, "At least one of the documents need to contains age equal to %s", someAge), - isEqualFound); - } - - @Test - public void orTest() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * " + - "FROM %s " + - "WHERE match_phrase(gender, 'F') OR match_phrase(gender, 'M') " + - "LIMIT 1000", TEST_INDEX_ACCOUNT)); - Assert.assertEquals(1000, getTotalHits(response)); - } - - @Test - public void andTest() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * FROM %s WHERE age=32 AND gender='M' LIMIT 1000", - TestsConstants.TEST_INDEX_PEOPLE)); - - JSONArray hits = getHits(response); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - Assert.assertEquals(32, getSource(hit).getInt("age")); - Assert.assertEquals("M", getSource(hit).getString("gender")); - } - } - - @Test - public void likeTest() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * FROM %s WHERE firstname LIKE 'amb%%' LIMIT 1000", - TEST_INDEX_ACCOUNT)); - - JSONArray hits = getHits(response); - Assert.assertEquals(1, getTotalHits(response)); - Assert.assertEquals("Amber", getSource(hits.getJSONObject(0)).getString("firstname")); - } - - @Test - public void notLikeTest() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * FROM %s WHERE firstname NOT LIKE 'amb%%'", - TEST_INDEX_ACCOUNT)); - - JSONArray hits = getHits(response); - Assert.assertNotEquals(0, getTotalHits(response)); - for (int i = 0; i < hits.length(); i ++) { - JSONObject hit = hits.getJSONObject(i); - Assert.assertFalse(getSource(hit).getString("firstname").toLowerCase().startsWith("amb")); - } - } - - @Test - public void regexQueryTest() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * " + - "FROM %s " + - "WHERE dog_name = REGEXP_QUERY('sn.*', 'INTERSECTION|COMPLEMENT|EMPTY', 10000)", - TestsConstants.TEST_INDEX_DOG)); - - JSONArray hits = getHits(response); - Assert.assertEquals(1, hits.length()); - - JSONObject hitSource = getSource(hits.getJSONObject(0)); - Assert.assertEquals("snoopy", hitSource.getString("dog_name")); - Assert.assertEquals("Hattie", hitSource.getString("holdersName")); - Assert.assertEquals(4, hitSource.getInt("age")); - } - - @Test - public void doubleNotTest() throws IOException { - JSONObject response1 = executeQuery( - String.format(Locale.ROOT, - "SELECT * FROM %s WHERE NOT gender LIKE 'm' AND NOT gender LIKE 'f'", - TEST_INDEX_ACCOUNT)); - Assert.assertEquals(0, getTotalHits(response1)); - - JSONObject response2 = executeQuery( - String.format(Locale.ROOT, - "SELECT * FROM %s WHERE NOT gender LIKE 'm' AND gender NOT LIKE 'f'", - TEST_INDEX_ACCOUNT)); - Assert.assertEquals(0, getTotalHits(response2)); - - JSONObject response3 = executeQuery( - String.format(Locale.ROOT, - "SELECT * FROM %s WHERE gender NOT LIKE 'm' AND gender NOT LIKE 'f'", - TEST_INDEX_ACCOUNT)); - Assert.assertEquals(0, getTotalHits(response3)); - - JSONObject response4 = executeQuery( - String.format(Locale.ROOT, - "SELECT * FROM %s WHERE gender LIKE 'm' AND NOT gender LIKE 'f'", - TEST_INDEX_ACCOUNT)); - // Assert there are results and they all have gender 'm' - Assert.assertNotEquals(0, getTotalHits(response4)); - JSONArray hits = getHits(response4); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - Assert.assertEquals("m", getSource(hit).getString("gender").toLowerCase()); - } - - JSONObject response5 = executeQuery( - String.format(Locale.ROOT, "SELECT * FROM %s WHERE NOT (gender = 'm' OR gender = 'f')", - TEST_INDEX_ACCOUNT)); - Assert.assertEquals(0, getTotalHits(response5)); - } - - @Test - public void limitTest() throws IOException { - JSONObject response = executeQuery(String.format(Locale.ROOT, "SELECT * FROM %s LIMIT 30", - TEST_INDEX_ACCOUNT)); - - JSONArray hits = getHits(response); - Assert.assertEquals(30, hits.length()); - } - - @Test - public void betweenTest() throws IOException { - int min = 27; - int max = 30; - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * FROM %s WHERE age BETWEEN %s AND %s LIMIT 1000", - TestsConstants.TEST_INDEX_PEOPLE, min, max)); - - JSONArray hits = getHits(response); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - int age = getSource(hit).getInt("age"); - assertThat(age, allOf(greaterThanOrEqualTo(min), lessThanOrEqualTo(max))); - } - } - - // TODO When using NOT BETWEEN on fields, documents not containing the field - // are returned as well. This may be incorrect behavior. - @Test - public void notBetweenTest() throws IOException { - int min = 20; - int max = 37; - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * FROM %s WHERE age NOT BETWEEN %s AND %s LIMIT 1000", - TestsConstants.TEST_INDEX_PEOPLE, min, max)); - - JSONArray hits = getHits(response); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - JSONObject source = getSource(hit); - - // Ignore documents which do not contain the age field - if (source.has("age")) { - int age = source.getInt("age"); - assertThat(age, not(allOf(greaterThanOrEqualTo(min), lessThanOrEqualTo(max)))); - } - } - } - - @Ignore("Semantic analysis failed because 'age' doesn't exist.") - @Test - public void inTest() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT age FROM %s WHERE age IN (20, 22) LIMIT 1000", - TestsConstants.TEST_INDEX_PHRASE)); - - JSONArray hits = getHits(response); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - int age = getSource(hit).getInt("age"); - assertThat(age, isOneOf(20, 22)); - } - } - - @Test - public void inTestWithStrings() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, - "SELECT phrase FROM %s WHERE phrase IN ('quick', 'fox') LIMIT 1000", - TestsConstants.TEST_INDEX_PHRASE)); - - JSONArray hits = getHits(response); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - String phrase = getSource(hit).getString("phrase"); - assertThat(phrase, isOneOf("quick fox here", "fox brown", "quick fox", "brown fox")); - } - } - - @Test - public void inTermsTestWithIdentifiersTreatedLikeStrings() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT name " + - "FROM %s " + - "WHERE name.firstname = IN_TERMS('daenerys','eddard') " + - "LIMIT 1000", - TestsConstants.TEST_INDEX_GAME_OF_THRONES)); - - JSONArray hits = getHits(response); - Assert.assertEquals(2, getTotalHits(response)); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - String firstname = ((JSONObject) getSource(hit).get("name")).getString("firstname"); - assertThat(firstname, isOneOf("Daenerys", "Eddard")); - } - } - - @Test - public void inTermsTestWithStrings() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT name " + - "FROM %s " + - "WHERE name.firstname = IN_TERMS('daenerys','eddard') " + - "LIMIT 1000", - TestsConstants.TEST_INDEX_GAME_OF_THRONES)); - - JSONArray hits = getHits(response); - Assert.assertEquals(2, getTotalHits(response)); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - String firstname = ((JSONObject) getSource(hit).get("name")).getString("firstname"); - assertThat(firstname, isOneOf("Daenerys", "Eddard")); - } - } - - @Test - public void inTermsWithNumbers() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT name " + - "FROM %s " + - "WHERE name.ofHisName = IN_TERMS(4,2) " + - "LIMIT 1000", - TestsConstants.TEST_INDEX_GAME_OF_THRONES)); - - JSONArray hits = getHits(response); - Assert.assertEquals(1, getTotalHits(response)); - - JSONObject hit = hits.getJSONObject(0); - String firstname = ((JSONObject) getSource(hit).get("name")).getString("firstname"); - Assert.assertEquals("Brandon", firstname); - } - - @Test - public void termQueryWithNumber() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, - "SELECT name FROM %s WHERE name.ofHisName = term(4) LIMIT 1000", - TestsConstants.TEST_INDEX_GAME_OF_THRONES)); - - JSONArray hits = getHits(response); - Assert.assertEquals(1, getTotalHits(response)); - - JSONObject hit = hits.getJSONObject(0); - String firstname = ((JSONObject) getSource(hit).get("name")).getString("firstname"); - Assert.assertEquals("Brandon", firstname); - } - - @Test - public void termQueryWithStringIdentifier() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT name " + - "FROM %s " + - "WHERE name.firstname = term('brandon') " + - "LIMIT 1000", - TestsConstants.TEST_INDEX_GAME_OF_THRONES)); - - JSONArray hits = getHits(response); - Assert.assertEquals(1, getTotalHits(response)); - - JSONObject hit = hits.getJSONObject(0); - String firstname = ((JSONObject) getSource(hit).get("name")).getString("firstname"); - Assert.assertEquals("Brandon", firstname); - } - - @Test - public void termQueryWithStringLiteral() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT name " + - "FROM %s " + - "WHERE name.firstname = term('brandon') " + - "LIMIT 1000", - TestsConstants.TEST_INDEX_GAME_OF_THRONES)); - - JSONArray hits = getHits(response); - Assert.assertEquals(1, getTotalHits(response)); - - JSONObject hit = hits.getJSONObject(0); - String firstname = ((JSONObject) getSource(hit).get("name")).getString("firstname"); - Assert.assertEquals("Brandon", firstname); - } - - // TODO When using NOT IN on fields, documents not containing the field - // are returned as well. This may be incorrect behavior. - @Test - public void notInTest() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT age FROM %s WHERE age NOT IN (20, 22) LIMIT 1000", - TestsConstants.TEST_INDEX_PEOPLE)); - - JSONArray hits = getHits(response); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - JSONObject source = getSource(hit); - - // Ignore documents which do not contain the age field - if (source.has("age")) { - int age = source.getInt("age"); - assertThat(age, not(isOneOf(20, 22))); - } - } - } - - @Test - public void dateSearch() throws IOException { - DateTimeFormatter formatter = DateTimeFormat.forPattern(TestsConstants.DATE_FORMAT); - DateTime dateToCompare = new DateTime(2014, 8, 18, 0, 0, 0); - - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT insert_time FROM %s WHERE insert_time < '2014-08-18'", - TestsConstants.TEST_INDEX_ONLINE)); - JSONArray hits = getHits(response); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - JSONObject source = getSource(hit); - DateTime insertTime = formatter.parseDateTime(source.getString("insert_time")); - - String errorMessage = String.format(Locale.ROOT, "insert_time must be before 2014-08-18. Found: %s", - insertTime); - Assert.assertTrue(errorMessage, insertTime.isBefore(dateToCompare)); - } - } - - @Test - public void dateSearchBraces() throws IOException { - DateTimeFormatter formatter = DateTimeFormat.forPattern(TestsConstants.TS_DATE_FORMAT); - DateTime dateToCompare = new DateTime(2015, 3, 15, 0, 0, 0); - - JSONObject response = executeQuery( - String.format(Locale.ROOT, - "SELECT odbc_time FROM %s/odbc WHERE odbc_time < {ts '2015-03-15 00:00:00.000'}", - TestsConstants.TEST_INDEX_ODBC)); - JSONArray hits = getHits(response); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - JSONObject source = getSource(hit); - String insertTimeStr = source.getString("odbc_time"); - insertTimeStr = insertTimeStr.replace("{ts '", "").replace("'}", ""); - - DateTime insertTime = formatter.parseDateTime(insertTimeStr); - String errorMessage = String.format(Locale.ROOT, "insert_time must be before 2015-03-15. Found: %s", - insertTime); - Assert.assertTrue(errorMessage, insertTime.isBefore(dateToCompare)); - } - } - - @Test - public void dateBetweenSearch() throws IOException { - DateTimeFormatter formatter = DateTimeFormat.forPattern(TestsConstants.DATE_FORMAT); - - DateTime dateLimit1 = new DateTime(2014, 8, 18, 0, 0, 0); - DateTime dateLimit2 = new DateTime(2014, 8, 21, 0, 0, 0); - - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT insert_time " + - "FROM %s " + - "WHERE insert_time BETWEEN '2014-08-18' AND '2014-08-21' " + - "LIMIT 3", - TestsConstants.TEST_INDEX_ONLINE)); - JSONArray hits = getHits(response); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - JSONObject source = getSource(hit); - DateTime insertTime = formatter.parseDateTime(source.getString("insert_time")); - - boolean isBetween = (insertTime.isAfter(dateLimit1) || insertTime.isEqual(dateLimit1)) && - (insertTime.isBefore(dateLimit2) || insertTime.isEqual(dateLimit2)); - - Assert.assertTrue("insert_time must be between 2014-08-18 and 2014-08-21", isBetween); - } - } - - @Test - public void missFilterSearch() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * FROM %s WHERE insert_time2 IS missing", - TestsConstants.TEST_INDEX_PHRASE)); - - JSONArray hits = getHits(response); - Assert.assertEquals(4, getTotalHits(response)); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - JSONObject source = getSource(hit); - - Assert.assertFalse(source.has("insert_time2")); - } - } - - @Test - public void notMissFilterSearch() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * FROM %s WHERE insert_time2 IS NOT missing", - TestsConstants.TEST_INDEX_PHRASE)); - - JSONArray hits = getHits(response); - Assert.assertEquals(2, getTotalHits(response)); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - JSONObject source = getSource(hit); - - Assert.assertTrue(source.has("insert_time2")); - } - } - - @Test - public void complexConditionQuery() throws IOException { - String errorMessage = "Result does not exist to the condition " + - "(gender='m' AND (age> 25 OR account_number>5)) OR (gender='f' AND (age>30 OR account_number < 8)"; - - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * " + - "FROM %s " + - "WHERE (gender='m' AND (age> 25 OR account_number>5)) " + - "OR (gender='f' AND (age>30 OR account_number < 8))", - TEST_INDEX_ACCOUNT)); - - JSONArray hits = getHits(response); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - JSONObject source = getSource(hit); - - String gender = source.getString("gender").toLowerCase(); - int age = source.getInt("age"); - int accountNumber = source.getInt("account_number"); - - Assert.assertTrue(errorMessage, - (gender.equals("m") && (age > 25 || accountNumber > 5)) - || (gender.equals("f") && (age > 30 || accountNumber < 8))); - } - } - - @Test - public void complexNotConditionQuery() throws IOException { - String errorMessage = "Result does not exist to the condition " + - "NOT (gender='m' AND NOT (age > 25 OR account_number > 5)) " + - "OR (NOT gender='f' AND NOT (age > 30 OR account_number < 8))"; - - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * " + - "FROM %s " + - "WHERE NOT (gender='m' AND NOT (age > 25 OR account_number > 5)) " + - "OR (NOT gender='f' AND NOT (age > 30 OR account_number < 8))", - TEST_INDEX_ACCOUNT)); - - JSONArray hits = getHits(response); - Assert.assertNotEquals(0, hits.length()); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - JSONObject source = getSource(hit); - - String gender = source.getString("gender").toLowerCase(); - int age = source.getInt("age"); - int accountNumber = source.getInt("account_number"); - - Assert.assertTrue(errorMessage, - !(gender.equals("m") && !(age > 25 || accountNumber > 5)) - || (!gender.equals("f") && !(age > 30 || accountNumber < 8))); - } - } - - @Test - @SuppressWarnings("unchecked") - public void orderByAscTest() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT age FROM %s ORDER BY age ASC LIMIT 1000", - TEST_INDEX_ACCOUNT)); - - JSONArray hits = getHits(response); - ArrayList ages = new ArrayList<>(); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - JSONObject source = getSource(hit); - - ages.add(source.getInt("age")); - } - - ArrayList sortedAges = (ArrayList) ages.clone(); - Collections.sort(sortedAges); - Assert.assertEquals("The list is not in ascending order", sortedAges, ages); - } - - @Test - public void orderByDescTest() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT age FROM %s ORDER BY age DESC LIMIT 1000", - TEST_INDEX_ACCOUNT)); - assertResponseForOrderByTest(response); - } - - @Test - public void orderByDescUsingTableAliasTest() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT a.age FROM %s a ORDER BY a.age DESC LIMIT 1000", - TEST_INDEX_ACCOUNT)); - assertResponseForOrderByTest(response); - } - - @SuppressWarnings("unchecked") - private void assertResponseForOrderByTest(JSONObject response) { - JSONArray hits = getHits(response); - ArrayList ages = new ArrayList<>(); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - JSONObject source = getSource(hit); - - ages.add(source.getInt("age")); - } - - ArrayList sortedAges = (ArrayList) ages.clone(); - Collections.sort(sortedAges, Collections.reverseOrder()); - Assert.assertEquals("The list is not in ascending order", sortedAges, ages); - } - - @Test - @SuppressWarnings("unchecked") - public void orderByAscFieldWithSpaceTest() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * " + - "FROM %s " + - "WHERE `test field` IS NOT null " + - "ORDER BY `test field` ASC " + - "LIMIT 1000", - TestsConstants.TEST_INDEX_PHRASE)); - - JSONArray hits = getHits(response); - ArrayList testFields = new ArrayList<>(); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - JSONObject source = getSource(hit); - - testFields.add(source.getInt("test field")); - } - - ArrayList sortedTestFields = (ArrayList) testFields.clone(); - Collections.sort(sortedTestFields); - Assert.assertEquals("The list is not in ascending order", sortedTestFields, testFields); - } - - @Test - public void testWhereWithBoolEqualsTrue() throws IOException { - JSONObject response = executeQuery( - StringUtils.format( - "SELECT * " + - "FROM %s " + - "WHERE male = true " + - "LIMIT 5", - TestsConstants.TEST_INDEX_BANK) - ); - - checkResponseSize(response, BANK_INDEX_MALE_TRUE); - } - - @Test - public void testWhereWithBoolEqualsTrueAndGroupBy() throws IOException { - JSONObject response = executeQuery( - StringUtils.format( - "SELECT * " + - "FROM %s " + - "WHERE male = true " + - "GROUP BY balance " + - "LIMIT 5", - TestsConstants.TEST_INDEX_BANK) - ); - - checkAggregationResponseSize(response, BANK_INDEX_MALE_TRUE); - } - - @Test - public void testWhereWithBoolEqualsTrueAndOrderBy() throws IOException { - JSONObject response = executeQuery( - StringUtils.format( - "SELECT * " + - "FROM %s " + - "WHERE male = true " + - "ORDER BY age " + - "LIMIT 5", - TestsConstants.TEST_INDEX_BANK) - ); - - checkResponseSize(response, BANK_INDEX_MALE_TRUE); - } - - @Test - public void testWhereWithBoolIsTrue() throws IOException { - JSONObject response = executeQuery( - StringUtils.format( - "SELECT * " + - "FROM %s " + - "WHERE male IS true " + - "GROUP BY balance " + - "LIMIT 5", - TestsConstants.TEST_INDEX_BANK) - ); - - checkAggregationResponseSize(response, BANK_INDEX_MALE_TRUE); - } - - @Test - public void testWhereWithBoolIsNotTrue() throws IOException { - JSONObject response = executeQuery( - StringUtils.format( - "SELECT * " + - "FROM %s " + - "WHERE male IS NOT true " + - "GROUP BY balance " + - "LIMIT 5", - TestsConstants.TEST_INDEX_BANK) - ); - - checkAggregationResponseSize(response, BANK_INDEX_MALE_FALSE); - } - - @Test - public void testWhereWithBoolEqualsFalse() throws IOException { - JSONObject response = executeQuery( - StringUtils.format( - "SELECT * " + - "FROM %s " + - "WHERE male = false " + - "LIMIT 5", - TestsConstants.TEST_INDEX_BANK) - ); - - checkResponseSize(response, BANK_INDEX_MALE_FALSE); - } - - @Test - public void testWhereWithBoolEqualsFalseAndGroupBy() throws IOException { - JSONObject response = executeQuery( - StringUtils.format( - "SELECT * " + - "FROM %s " + - "WHERE male = false " + - "GROUP BY balance " + - "LIMIT 5", - TestsConstants.TEST_INDEX_BANK) - ); - - checkAggregationResponseSize(response, BANK_INDEX_MALE_FALSE); - } - - @Test - public void testWhereWithBoolEqualsFalseAndOrderBy() throws IOException { - JSONObject response = executeQuery( - StringUtils.format( - "SELECT * " + - "FROM %s " + - "WHERE male = false " + - "ORDER BY age " + - "LIMIT 5", - TestsConstants.TEST_INDEX_BANK) - ); - - checkResponseSize(response, BANK_INDEX_MALE_FALSE); - } - - @Test - public void testWhereWithBoolIsFalse() throws IOException { - JSONObject response = executeQuery( - StringUtils.format( - "SELECT * " + - "FROM %s " + - "WHERE male IS false " + - "GROUP BY balance " + - "LIMIT 5", - TestsConstants.TEST_INDEX_BANK) - ); - - checkAggregationResponseSize(response, BANK_INDEX_MALE_FALSE); - } - - @Test - public void testWhereWithBoolIsNotFalse() throws IOException { - JSONObject response = executeQuery( - StringUtils.format( - "SELECT * " + - "FROM %s " + - "WHERE male IS NOT false " + - "GROUP BY balance " + - "LIMIT 5", - TestsConstants.TEST_INDEX_BANK) - ); - - checkAggregationResponseSize(response, BANK_INDEX_MALE_TRUE); - } - - @Test - public void testMultiPartWhere() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * " + - "FROM %s " + - "WHERE (firstname LIKE 'opal' OR firstname LIKE 'rodriquez') " + - "AND (state like 'oh' OR state like 'hi')", - TEST_INDEX_ACCOUNT)); - - Assert.assertEquals(2, getTotalHits(response)); - } - - @Test - public void testMultiPartWhere2() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * " + - "FROM %s " + - "WHERE ((account_number > 200 AND account_number < 300) OR gender LIKE 'm') " + - "AND (state LIKE 'hi' OR address LIKE 'avenue')", - TEST_INDEX_ACCOUNT)); - - Assert.assertEquals(127, getTotalHits(response)); - } - - @Test - public void testMultiPartWhere3() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * " + - "FROM %s " + - "WHERE ((account_number > 25 AND account_number < 75) AND age >35 ) " + - "AND (state LIKE 'md' OR (address LIKE 'avenue' OR address LIKE 'street'))", - TEST_INDEX_ACCOUNT)); - - Assert.assertEquals(7, getTotalHits(response)); - } - - @Test - public void filterPolygonTest() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * " + - "FROM %s " + - "WHERE GEO_INTERSECTS(place,'POLYGON ((102 2, 103 2, 103 3, 102 3, 102 2))')", - TestsConstants.TEST_INDEX_LOCATION)); - - JSONArray hits = getHits(response); - Assert.assertEquals(1, getTotalHits(response)); - - JSONObject hit = hits.getJSONObject(0); - Assert.assertEquals("bigSquare", getSource(hit).getString("description")); - } - - @Test - public void boundingBox() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, - "SELECT * FROM %s WHERE GEO_BOUNDING_BOX(center, 100.0, 1.0, 101, 0.0)", - TestsConstants.TEST_INDEX_LOCATION)); - - JSONArray hits = getHits(response); - Assert.assertEquals(1, getTotalHits(response)); - - JSONObject hit = hits.getJSONObject(0); - Assert.assertEquals("square", getSource(hit).getString("description")); - } - - @Test - public void geoDistance() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, - "SELECT * FROM %s WHERE GEO_DISTANCE(center, '1km', 100.5, 0.500001)", - TestsConstants.TEST_INDEX_LOCATION)); - - JSONArray hits = getHits(response); - Assert.assertEquals(1, getTotalHits(response)); - - JSONObject hit = hits.getJSONObject(0); - Assert.assertEquals("square", getSource(hit).getString("description")); - } - - @Test - public void geoPolygon() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, - "SELECT * FROM %s WHERE GEO_POLYGON(center, 100,0, 100.5, 2, 101.0,0)", - TestsConstants.TEST_INDEX_LOCATION)); - - JSONArray hits = getHits(response); - Assert.assertEquals(1, getTotalHits(response)); - - JSONObject hit = hits.getJSONObject(0); - Assert.assertEquals("square", getSource(hit).getString("description")); - } - - @Ignore - @Test - public void escapedCharactersCheck() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * " + - "FROM %s " + - "WHERE MATCH_PHRASE(nickname, 'Daenerys \"Stormborn\"') " + - "LIMIT 1000", - TestsConstants.TEST_INDEX_GAME_OF_THRONES)); - - Assert.assertEquals(1, getTotalHits(response)); - } - - @Test - public void complexObjectSearch() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * " + - "FROM %s " + - "WHERE MATCH_PHRASE(name.firstname, 'Jaime') " + - "LIMIT 1000", - TestsConstants.TEST_INDEX_GAME_OF_THRONES)); - - Assert.assertEquals(1, getTotalHits(response)); - } - - @Test - public void complexObjectReturnField() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT parents.father " + - "FROM %s " + - "WHERE MATCH_PHRASE(name.firstname, 'Brandon') " + - "LIMIT 1000", - TestsConstants.TEST_INDEX_GAME_OF_THRONES)); - - JSONArray hits = getHits(response); - Assert.assertEquals(1, getTotalHits(response)); - - JSONObject hit = hits.getJSONObject(0); - Assert.assertEquals("Eddard", getSource(hit).getJSONObject("parents").getString("father")); - } - - /** - * TODO: Fields prefixed with @ gets converted to SQLVariantRefExpr instead of SQLIdentifierExpr - * Either change SQLVariantRefExpr to SQLIdentifierExpr - * Or handle the special case for SQLVariantRefExpr - */ - @Ignore - @Test - public void queryWithAtFieldOnWhere() throws IOException { - JSONObject response = executeQuery(String.format(Locale.ROOT, - "SELECT * FROM %s where @wolf = 'Summer' LIMIT 1000", TEST_INDEX_GAME_OF_THRONES)); - Assert.assertEquals(1, getTotalHits(response)); - JSONObject hit = getHits(response).getJSONObject(0); - Assert.assertEquals("Summer", hit.get("@wolf")); - Assert.assertEquals("Brandon", hit.query("name/firstname")); - } - - @Test - public void queryWithDotAtStartOfIndexName() throws Exception { - TestUtils.createHiddenIndexByRestClient(client(), ".bank", null); - TestUtils.loadDataByRestClient(client(), ".bank", "/src/test/resources/.bank.json"); - - String response = executeQuery("SELECT education FROM .bank WHERE account_number = 12345", - "jdbc"); - Assert.assertTrue(response.contains("PhD")); - } - - @Test - public void notLikeTests() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT name " + - "FROM %s " + - "WHERE name.firstname NOT LIKE 'd%%' AND name IS NOT NULL " + - "LIMIT 1000", - TestsConstants.TEST_INDEX_GAME_OF_THRONES)); - - JSONArray hits = getHits(response); - Assert.assertEquals(3, getTotalHits(response)); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - JSONObject source = getSource(hit); - - String name = source.getJSONObject("name").getString("firstname"); - Assert.assertFalse(String.format(Locale.ROOT, "Name [%s] should not match pattern [d%%]", name), - name.startsWith("d")); - } - } - - @Test - public void isNullTest() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT name " + - "FROM %s " + - "WHERE nickname IS NULL " + - "LIMIT 1000", - TestsConstants.TEST_INDEX_GAME_OF_THRONES)); - - Assert.assertEquals(6, getTotalHits(response)); - } - - @Test - public void isNotNullTest() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT name " + - "FROM %s " + - "WHERE nickname IS NOT NULL " + - "LIMIT 1000", - TestsConstants.TEST_INDEX_GAME_OF_THRONES)); - - Assert.assertEquals(1, getTotalHits(response)); - } - - @Test - public void innerQueryTest() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * " + - "FROM %s D " + - "WHERE holdersName IN (SELECT firstname " + - "FROM %s " + - "WHERE firstname = 'Hattie')", - TestsConstants.TEST_INDEX_DOG, TEST_INDEX_ACCOUNT)); - - JSONArray hits = getHits(response); - Assert.assertEquals(1, hits.length()); - - JSONObject hit = hits.getJSONObject(0); - JSONObject source = getSource(hit); - Assert.assertEquals("snoopy", source.getString("D.dog_name")); - Assert.assertEquals("Hattie", source.getString("D.holdersName")); - Assert.assertEquals(4, source.getInt("D.age")); - } - - @Ignore - @Test - public void twoSubQueriesTest() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * " + - "FROM %s " + - "WHERE holdersName IN (SELECT firstname " + - "FROM %s " + - "WHERE firstname = 'Hattie') " + - "AND age IN (SELECT name.ofHisName " + - "FROM %s " + - "WHERE name.firstname <> 'Daenerys' " + - "AND name.ofHisName IS NOT NULL) ", - TestsConstants.TEST_INDEX_DOG, - TEST_INDEX_ACCOUNT, - TestsConstants.TEST_INDEX_GAME_OF_THRONES)); - - JSONArray hits = getHits(response); - Assert.assertEquals(1, hits.length()); - - JSONObject hit = hits.getJSONObject(0); - JSONObject source = getSource(hit); - Assert.assertEquals("snoopy", source.getString("dog_name")); - Assert.assertEquals("Hattie", source.getString("holdersName")); - Assert.assertEquals(4, source.getInt("age")); - } - - @Ignore - @Test - public void inTermsSubQueryTest() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * " + - "FROM %s " + - "WHERE age = IN_TERMS (SELECT name.ofHisName " + - "FROM %s " + - "WHERE name.firstname <> 'Daenerys' " + - "AND name.ofHisName IS NOT NULL)", - TestsConstants.TEST_INDEX_DOG, TestsConstants.TEST_INDEX_GAME_OF_THRONES)); - - JSONArray hits = getHits(response); - Assert.assertEquals(1, hits.length()); - - JSONObject hit = hits.getJSONObject(0); - JSONObject source = getSource(hit); - Assert.assertEquals("snoopy", source.getString("dog_name")); - Assert.assertEquals("Hattie", source.getString("holdersName")); - Assert.assertEquals(4, source.getInt("age")); - } - - @Ignore - @Test - public void idsQueryOneId() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * FROM %s WHERE _id = IDS_QUERY(dog, 1)", - TestsConstants.TEST_INDEX_DOG)); - - JSONArray hits = getHits(response); - Assert.assertEquals(1, hits.length()); - - JSONObject hit = hits.getJSONObject(0); - JSONObject source = getSource(hit); - Assert.assertEquals("rex", source.getString("dog_name")); - Assert.assertEquals("Daenerys", source.getString("holdersName")); - Assert.assertEquals(2, hit.getInt("age")); - } - - @Ignore - @Test - public void idsQueryMultipleId() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * FROM %s WHERE _id = IDS_QUERY(dog, 1, 2, 3)", - TestsConstants.TEST_INDEX_DOG)); - - JSONArray hits = getHits(response); - Assert.assertEquals(1, hits.length()); - - JSONObject hit = hits.getJSONObject(0); - JSONObject source = getSource(hit); - Assert.assertEquals("rex", source.getString("dog_name")); - Assert.assertEquals("Daenerys", source.getString("holdersName")); - Assert.assertEquals(2, hit.getInt("age")); - } - - @Ignore - @Test - public void idsQuerySubQueryIds() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * " + - "FROM %s " + - "WHERE _id = IDS_QUERY(dog, (SELECT name.ofHisName " + - "FROM %s " + - "WHERE name.firstname <> 'Daenerys' " + - "AND name.ofHisName IS NOT NULL))", - TestsConstants.TEST_INDEX_DOG, TestsConstants.TEST_INDEX_GAME_OF_THRONES)); - - JSONArray hits = getHits(response); - Assert.assertEquals(1, hits.length()); - - JSONObject hit = hits.getJSONObject(0); - JSONObject source = getSource(hit); - Assert.assertEquals("rex", source.getString("dog_name")); - Assert.assertEquals("Daenerys", source.getString("holdersName")); - Assert.assertEquals(2, hit.getInt("age")); - } - - @Test - public void nestedEqualsTestFieldNormalField() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * FROM %s WHERE nested(message.info)='b'", - TestsConstants.TEST_INDEX_NESTED_TYPE)); - - Assert.assertEquals(1, getTotalHits(response)); - } - - @Test - public void nestedEqualsTestFieldInsideArrays() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * FROM %s WHERE nested(message.info) = 'a'", - TestsConstants.TEST_INDEX_NESTED_TYPE)); - - Assert.assertEquals(2, getTotalHits(response)); - } - - @Ignore // Seems like we don't support nested with IN, throwing IllegalArgumentException - @Test - public void nestedOnInQuery() throws IOException { - JSONObject response = executeQuery(String.format(Locale.ROOT, - "SELECT * FROM %s where nested(message.info) IN ('a','b')", TEST_INDEX_NESTED_TYPE)); - Assert.assertEquals(3, getTotalHits(response)); - } - - @Test - public void complexNestedQueryBothOnSameObject() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * " + - "FROM %s " + - "WHERE nested('message', message.info = 'a' AND message.author ='i')", - TestsConstants.TEST_INDEX_NESTED_TYPE)); - - Assert.assertEquals(1, getTotalHits(response)); - } - - @Test - public void complexNestedQueryNotBothOnSameObject() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * " + - "FROM %s " + - "WHERE nested('message', message.info = 'a' AND message.author ='h')", - TestsConstants.TEST_INDEX_NESTED_TYPE)); - - Assert.assertEquals(0, getTotalHits(response)); - } - - @Test - public void nestedOnInTermsQuery() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT * " + - "FROM %s " + - "WHERE nested(message.info) = IN_TERMS('a', 'b')", - TestsConstants.TEST_INDEX_NESTED_TYPE)); - - Assert.assertEquals(3, getTotalHits(response)); - } - - // TODO Uncomment these after problem with loading join index is resolved -// @Test -// public void childrenEqualsTestFieldNormalField() throws IOException { -// JSONObject response = executeQuery( -// String.format(Locale.ROOT, "SELECT * " + -// "FROM %s/joinType " + -// "WHERE children(childrenType, info) = 'b'", TestsConstants.TEST_INDEX_JOIN_TYPE)); -// -// Assert.assertEquals(1, getTotalHits(response)); -// } -// -// @Test -// public void childrenOnInQuery() throws IOException { -// JSONObject response = executeQuery( -// String.format(Locale.ROOT, "SELECT * " + -// "FROM %s/joinType " + -// "WHERE children(childrenType, info) IN ('a', 'b')", -// TestsConstants.TEST_INDEX_JOIN_TYPE)); -// -// Assert.assertEquals(2, getTotalHits(response)); -// } -// -// @Test -// public void complexChildrenQueryBothOnSameObject() throws IOException { -// JSONObject response = executeQuery( -// String.format(Locale.ROOT, "SELECT * " + -// "FROM %s/joinType " + -// "WHERE children(childrenType, info = 'a' AND author ='e')", -// TestsConstants.TEST_INDEX_JOIN_TYPE)); -// -// Assert.assertEquals(1, getTotalHits(response)); -// } -// -// @Test -// public void complexChildrenQueryNotOnSameObject() throws IOException { -// JSONObject response = executeQuery( -// String.format(Locale.ROOT, "SELECT * " + -// "FROM %s/joinType " + -// "WHERE children(childrenType, info = 'a' AND author ='j')", -// TestsConstants.TEST_INDEX_JOIN_TYPE)); -// -// Assert.assertEquals(0, getTotalHits(response)); -// } -// -// @Test -// public void childrenOnInTermsQuery() throws IOException { -// JSONObject response = executeQuery( -// String.format(Locale.ROOT, "SELECT * " + -// "FROM %s/joinType " + -// "WHERE children(childrenType, info) = IN_TERMS(a, b)", -// TestsConstants.TEST_INDEX_JOIN_TYPE)); -// -// Assert.assertEquals(2, getTotalHits(response)); -// } - - @Ignore // the hint does not really work, NoSuchIndexException is thrown - @Test - public void multipleIndicesOneNotExistWithHint() throws IOException { - - JSONObject response = executeQuery(String.format(Locale.ROOT, "SELECT /*! IGNORE_UNAVAILABLE */ * FROM %s,%s ", TEST_INDEX_ACCOUNT,"badindex")); - - Assert.assertTrue(getTotalHits(response) > 0); - } - - @Test - public void multipleIndicesOneNotExistWithoutHint() throws IOException { - try { - executeQuery( - String.format(Locale.ROOT, "SELECT * FROM %s, %s", TEST_INDEX_ACCOUNT, "badindex")); - Assert.fail("Expected exception, but call succeeded"); - } catch (ResponseException e) { - Assert.assertEquals(RestStatus.BAD_REQUEST.getStatus(), e.getResponse().getStatusLine().getStatusCode()); - final String entity = TestUtils.getResponseBody(e.getResponse()); - Assert.assertThat(entity, containsString("\"type\": \"IndexNotFoundException\"")); - } - } - - // TODO Find way to check routing() without SearchRequestBuilder - // to properly update these tests to ESIntegTestCase format -// @Test -// public void routingRequestOneRounting() throws IOException { -// SqlElasticSearchRequestBuilder request = getRequestBuilder(String.format(Locale.ROOT, -// "SELECT /*! ROUTINGS(hey) */ * FROM %s ", TEST_INDEX_ACCOUNT)); -// SearchRequestBuilder searchRequestBuilder = (SearchRequestBuilder) request.getBuilder(); -// Assert.assertEquals("hey",searchRequestBuilder.request().routing()); -// } -// -// @Test -// public void routingRequestMultipleRountings() throws IOException { -// SqlElasticSearchRequestBuilder request = getRequestBuilder(String.format(Locale.ROOT, -// "SELECT /*! ROUTINGS(hey,bye) */ * FROM %s ", TEST_INDEX_ACCOUNT)); -// SearchRequestBuilder searchRequestBuilder = (SearchRequestBuilder) request.getBuilder(); -// Assert.assertEquals("hey,bye",searchRequestBuilder.request().routing()); -// } - - @Ignore // Getting parser error: syntax error, expect RPAREN, actual IDENTIFIER insert_time - @Test - public void scriptFilterNoParams() throws IOException { - - JSONObject result = executeQuery(String.format(Locale.ROOT, - "SELECT insert_time FROM %s where script('doc[\\'insert_time\''].date.hourOfDay==16') " + - "and insert_time <'2014-08-21T00:00:00.000Z'", TEST_INDEX_ONLINE)); - Assert.assertEquals(237, getTotalHits(result)); - } - - @Ignore // Getting parser error: syntax error, expect RPAREN, actual IDENTIFIER insert_time - @Test - public void scriptFilterWithParams() throws IOException { - - JSONObject result = executeQuery(String.format(Locale.ROOT, - "SELECT insert_time FROM %s where script('doc[\\'insert_time\''].date.hourOfDay==x','x'=16) " + - "and insert_time <'2014-08-21T00:00:00.000Z'", TEST_INDEX_ONLINE)); - Assert.assertEquals(237, getTotalHits(result)); - } - - @Test - public void highlightPreTagsAndPostTags() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, - "SELECT /*! HIGHLIGHT(phrase, pre_tags : [''], post_tags : ['']) */ " + - "* FROM %s " + - "WHERE phrase LIKE 'fox' " + - "ORDER BY _score", TestsConstants.TEST_INDEX_PHRASE)); - - JSONArray hits = getHits(response); - for (int i = 0; i < hits.length(); i++) { - JSONObject hit = hits.getJSONObject(i); - JSONObject highlightFields = hit.getJSONObject("highlight"); - - String phrase = highlightFields.getJSONArray("phrase").getString(0); - Assert.assertTrue(phrase.contains("fox")); - } - } - - @Ignore - @Test - public void fieldCollapsingTest() throws IOException { - JSONObject response = executeQuery( - String.format(Locale.ROOT, "SELECT /*! COLLAPSE({\"field\":\"age\"," + - "\"inner_hits\":{\"name\": \"account\"," + - "\"size\":1," + - "\"sort\":[{\"age\":\"asc\"}]}," + - "\"max_concurrent_group_searches\": 4}) */ " + - "* FROM %s", TEST_INDEX_ACCOUNT)); - - JSONArray hits = getHits(response); - Assert.assertEquals(21, hits.length()); - } - - @Test - public void backticksQuotedIndexNameTest() throws Exception { - TestUtils.createIndexByRestClient(client(), "bank_unquote", null); - TestUtils.loadDataByRestClient(client(), "bank", "/src/test/resources/bank_for_unquote_test.json"); - - JSONArray hits = getHits(executeQuery("SELECT lastname FROM `bank`")); - Object responseIndex = ((JSONObject) hits.get(0)).query("/_index"); - assertEquals("bank", responseIndex); - - assertEquals( - executeQuery("SELECT lastname FROM bank", "jdbc"), - executeQuery("SELECT `bank`.`lastname` FROM `bank`", "jdbc") - ); - - assertEquals( - executeQuery( - "SELECT `b`.`age` AS `AGE`, AVG(`b`.`balance`) FROM `bank` AS `b` " + - "WHERE ABS(`b`.`age`) > 20 GROUP BY `b`.`age` ORDER BY `b`.`age`", - "jdbc"), - executeQuery("SELECT b.age AS AGE, AVG(balance) FROM bank AS b " + - "WHERE ABS(age) > 20 GROUP BY b.age ORDER BY b.age", - "jdbc") - ); - } - - @Test - public void backticksQuotedFieldNamesTest() { - String expected = executeQuery(StringUtils.format("SELECT b.lastname FROM %s " + - "AS b ORDER BY age LIMIT 3", TestsConstants.TEST_INDEX_BANK), "jdbc"); - String quotedFieldResult = executeQuery(StringUtils.format("SELECT b.`lastname` FROM %s " + - "AS b ORDER BY age LIMIT 3", TestsConstants.TEST_INDEX_BANK), "jdbc"); - - assertEquals(expected, quotedFieldResult); - } - - @Test - public void backticksQuotedAliasTest() { - String expected = executeQuery(StringUtils.format("SELECT b.lastname FROM %s " + - "AS b ORDER BY age LIMIT 3", TestsConstants.TEST_INDEX_BANK), "jdbc"); - String quotedAliasResult = executeQuery(StringUtils.format("SELECT `b`.lastname FROM %s" + - " AS `b` ORDER BY age LIMIT 3", TestsConstants.TEST_INDEX_BANK), "jdbc"); - String quotedAliasAndFieldResult = executeQuery(StringUtils.format("SELECT `b`.`lastname` FROM %s " + - "AS `b` ORDER BY age LIMIT 3", TestsConstants.TEST_INDEX_BANK), "jdbc"); - - assertEquals(expected, quotedAliasResult); - assertEquals(expected, quotedAliasAndFieldResult); - } - - @Test - public void backticksQuotedAliasWithSpecialCharactersTest() { - String expected = executeQuery(StringUtils.format("SELECT b.lastname FROM %s " + - "AS b ORDER BY age LIMIT 3", TestsConstants.TEST_INDEX_BANK), "jdbc"); - String specialCharAliasResult = executeQuery(StringUtils.format("SELECT `b k`.lastname FROM %s " + - "AS `b k` ORDER BY age LIMIT 3", TestsConstants.TEST_INDEX_BANK), "jdbc"); - - assertEquals(expected, specialCharAliasResult); - } - - @Test - public void backticksQuotedAliasInJDBCResponseTest() { - String query = StringUtils.format("SELECT `b`.`lastname` AS `name` FROM %s AS `b` " + - "ORDER BY age LIMIT 3", TestsConstants.TEST_INDEX_BANK); - String response = executeQuery(query, "jdbc"); - - assertTrue(response.contains("\"alias\": \"name\"")); - } - - @Test - public void caseWhenSwitchTest() throws IOException { - JSONObject response = executeQuery("SELECT CASE age " + - "WHEN '30' THEN '1' " + - "WHEN '40' THEN '2' " + - "ELSE '0' END AS cases FROM " + TEST_INDEX_ACCOUNT + " WHERE age IS NOT NULL"); - JSONObject hit = getHits(response).getJSONObject(0); - String age = hit.query("/_source/age").toString(); - String cases = age.equals("30") ? "1" : age.equals("40") ? "2" : "0"; - - assertThat(cases, equalTo(hit.query("/fields/cases/0"))); - } - - @Test - public void caseWhenJdbcResponseTest() { - String response = executeQuery("SELECT CASE age " + - "WHEN '30' THEN 'age is 30' " + - "WHEN '40' THEN 'age is 40' " + - "ELSE 'NA' END AS cases FROM " + TEST_INDEX_ACCOUNT + " WHERE age is not null", "jdbc"); - assertTrue( - response.contains("age is 30") || - response.contains("age is 40") || - response.contains("NA") - ); - } - - @Test - public void functionInCaseFieldShouldThrowESExceptionDueToIllegalScriptInJdbc() { - String response = executeQuery( - "select case lower(firstname) when 'amber' then '1' else '2' end as cases from " + TEST_INDEX_ACCOUNT, - "jdbc"); - queryInJdbcResponseShouldIndicateESException(response, "SearchPhaseExecutionException", - "For more details, please send request for Json format"); - } - - @Test - public void functionCallWithIllegalScriptShouldThrowESExceptionInJdbc() { - String response = executeQuery("select log(balance + 2) from " + TEST_INDEX_BANK, - "jdbc"); - queryInJdbcResponseShouldIndicateESException(response, "SearchPhaseExecutionException", - "please send request for Json format to see the raw response from elasticsearch engine."); - } - - @Ignore("Goes in different route, does not call PrettyFormatRestExecutor.execute methods." + - "The performRequest method in RestClient doesn't throw any exceptions for null value fields in script") - @Test - public void functionArgWithNullValueFieldShouldThrowESExceptionInJdbc() { - String response = executeQuery( - "select log(balance) from " + TEST_INDEX_BANK_WITH_NULL_VALUES, "jdbc"); - queryInJdbcResponseShouldIndicateESException(response, "SearchPhaseExecutionException", - "For more details, please send request for Json format"); - } - - private void queryInJdbcResponseShouldIndicateESException(String response, String exceptionType, String... errMsgs) { - Assert.assertThat(response, containsString(exceptionType)); - for (String errMsg: errMsgs) { - Assert.assertThat(response, containsString(errMsg)); - } - } - - private String getScrollId(JSONObject response) { - return response.getString("_scroll_id"); - } - - private void checkResponseSize(JSONObject response, int sizeCheck) { - JSONArray queryResponse = getHits(response); - Assert.assertThat(queryResponse.length(), equalTo(sizeCheck)); - } - - private void checkAggregationResponseSize(JSONObject response, int sizeCheck) { - JSONArray queryResponse = (JSONArray)response.query("/aggregations/balance/buckets"); - Assert.assertThat(queryResponse.length(), equalTo(sizeCheck)); - } - - private void checkSelectAllAndFieldResponseSize(JSONObject response) { - String[] arr = new String[] {"account_number", "firstname", "address", "birthdate", "gender", "city", "lastname", - "balance", "employer", "state", "age", "email", "male"}; - Set expectedSource = new HashSet<>(Arrays.asList(arr)); - - JSONArray hits = getHits(response); - Assert.assertTrue(hits.length() > 0); - for (int i = 0; i < hits.length(); ++i) { - JSONObject hit = hits.getJSONObject(i); - Assert.assertEquals(expectedSource, getSource(hit).keySet()); - } - } - - private void checkSelectAllAndFieldAggregationResponseSize(JSONObject response, String field) { - JSONObject fieldAgg = (response.getJSONObject("aggregations")).getJSONObject(field); - JSONArray buckets = fieldAgg.getJSONArray("buckets"); - Assert.assertTrue(buckets.length() == 6); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLFunctionsIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLFunctionsIT.java deleted file mode 100644 index c8019c55c2..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLFunctionsIT.java +++ /dev/null @@ -1,844 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHits; -import org.hamcrest.collection.IsMapContaining; -import org.json.JSONObject; -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.IOException; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Locale; -import java.util.TimeZone; -import java.util.stream.IntStream; - -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_ACCOUNT; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.hitAny; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.kvDouble; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.kvInt; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.kvString; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.rows; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.schema; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyDataRows; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifySchema; -import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.both; -import static org.hamcrest.Matchers.closeTo; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.endsWith; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasEntry; -import static org.hamcrest.Matchers.hasValue; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.isEmptyOrNullString; -import static org.hamcrest.Matchers.not; - - -/** - * Created by allwefantasy on 8/25/16. - */ -public class SQLFunctionsIT extends SQLIntegTestCase { - - @Override - protected void init() throws Exception { - loadIndex(Index.ACCOUNT); - loadIndex(Index.ONLINE); - loadIndex(Index.DATE); - } - - @Test - public void functionFieldAliasAndGroupByAlias() throws Exception { - String query = "SELECT " + - "floor(substring(address,0,3)*20) as key," + - "sum(age) cvalue FROM " + TEST_INDEX_ACCOUNT + " where address is not null " + - "group by key order by cvalue desc limit 10 "; - final JSONObject result = executeQuery(query); - - - IntStream.rangeClosed(0, 9).forEach(i -> { - Assert.assertNotNull(result.query(String.format("/aggregations/key/buckets/%d/key", i))); - Assert.assertNotNull(result.query(String.format("/aggregations/key/buckets/%d/cvalue/value", i))); - } - ); - } - - /** - * todo fix the issue. - * @see https://github.com/opendistro-for-elasticsearch/sql/issues/59 - */ - @Ignore - public void normalFieldAlias() throws Exception { - - //here is a bug,csv field with spa - String query = "SELECT " + - "address as key,age from " + - TEST_INDEX_ACCOUNT + " where address is not null " + - "limit 10 "; - - assertThat( - executeQuery(query), - hitAny(kvString("/_source/key", not(isEmptyOrNullString()))) - ); - } - - @Test - public void functionAlias() throws Exception { - //here is a bug,if only script fields are included,then all fields will return; fix later - String query = "SELECT " + - "substring(address,0,3) as key,address from " + - TEST_INDEX_ACCOUNT + " where address is not null " + - "order by address desc limit 10 "; - - assertThat( - executeQuery(query), - hitAny(both(kvString("/_source/address", equalTo("863 Wythe Place"))).and(kvString("/fields/key/0", - equalTo("863")))) - ); - } - - @Test - public void caseChangeTest() throws IOException { - String query = "SELECT LOWER(firstname) " + - "FROM elasticsearch-sql_test_index_account " + - "WHERE UPPER(lastname)='DUKE' " + - "ORDER BY upper(lastname) "; - - assertThat( - executeQuery(query), - hitAny( - kvString("/_source/address", equalTo("880 Holmes Lane")), - kvString("/fields/LOWER(firstname)/0", equalTo("amber"))) - ); - } - - @Test - public void caseChangeTestWithLocale() throws IOException { - // Uses Turkish locale to check if we pass correct locale for case changing functions - // "IL".toLowerCase() in a Turkish locale returns "ıl" - // https://stackoverflow.com/questions/11063102/using-locales-with-javas-tolowercase-and-touppercase - - String query = "SELECT LOWER(state.keyword, 'tr') " + - "FROM elasticsearch-sql_test_index_account " + - "WHERE account_number=1"; - - assertThat( - executeQuery(query), - hitAny( - kvString("/fields/LOWER(state.keyword, 'tr')/0", equalTo("ıl"))) - ); - } - - @Test - public void caseChangeWithAggregationTest() throws IOException { - String query = "SELECT UPPER(e.firstname) AS upper, COUNT(*)" + - "FROM elasticsearch-sql_test_index_account e " + - "WHERE LOWER(e.lastname)='duke' " + - "GROUP BY upper"; - - assertThat( - executeQuery(query), - hitAny("/aggregations/upper/buckets", kvString("/key", equalTo("AMBER")))); - } - - @Test - public void castIntFieldToDoubleWithoutAliasTest() throws IOException { - String query = "SELECT CAST(age AS DOUBLE) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + - " ORDER BY age DESC LIMIT 5"; - - SearchHit[] hits = query(query).getHits(); - checkSuccessfulFieldCast(hits, "cast_age", "DOUBLE"); - for (int i = 0; i < hits.length; ++i) { - Assert.assertThat(hits[i].getFields().get("cast_age").getValue(), is(40.0)); - } - } - - @Test - public void castIntFieldToDoubleWithAliasTest() throws IOException { - String query = "SELECT CAST(age AS DOUBLE) AS test_alias FROM " + TestsConstants.TEST_INDEX_ACCOUNT + - " ORDER BY age LIMIT 5"; - - SearchHit[] hits = query(query).getHits(); - checkSuccessfulFieldCast(hits, "test_alias", "DOUBLE"); - for (int i = 0; i < hits.length; ++i) { - Assert.assertThat(hits[i].getFields().get("test_alias").getValue(), is(20.0)); - } - } - - @Test - public void castIntFieldToStringWithoutAliasTest() throws IOException { - String query = "SELECT CAST(balance AS STRING) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + - " ORDER BY balance LIMIT 1"; - - SearchHit[] hits = query(query).getHits(); - checkSuccessfulFieldCast(hits, "cast_balance", "STRING"); - for (int i = 0; i < hits.length; ++i) { - Assert.assertThat(hits[i].getFields().get("cast_balance").getValue(), is("1011")); - } - } - - @Test - public void castIntFieldToStringWithAliasTest() throws IOException { - String query = "SELECT CAST(balance AS STRING) AS cast_string_alias FROM " + TestsConstants.TEST_INDEX_ACCOUNT + - " ORDER BY cast_string_alias DESC LIMIT 1"; - - SearchHit[] hits = query(query).getHits(); - checkSuccessfulFieldCast(hits, "cast_string_alias", "STRING"); - for (int i = 0; i < hits.length; ++i) { - Assert.assertThat(hits[i].getFields().get("cast_string_alias").getValue(), is("9838")); - } - - } - - @Test - public void castIntFieldToFloatWithoutAliasJdbcFormatTest() { - JSONObject response = executeJdbcRequest( - "SELECT CAST(balance AS FLOAT) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + - " ORDER BY balance DESC LIMIT 1"); - - verifySchema(response, - schema("cast_balance", null, "float")); - - verifyDataRows(response, - rows(49989)); - } - - @Test - public void castIntFieldToFloatWithAliasJdbcFormatTest() { - JSONObject response = executeJdbcRequest( - "SELECT CAST(balance AS FLOAT) AS jdbc_float_alias " + - "FROM " + TestsConstants.TEST_INDEX_ACCOUNT + " ORDER BY jdbc_float_alias LIMIT 1"); - - verifySchema(response, - schema("jdbc_float_alias", null, "float")); - - verifyDataRows(response, - rows(1011)); - } - - @Test - public void castIntFieldToDoubleWithoutAliasOrderByTest() throws IOException { - String query = "SELECT CAST(age AS DOUBLE) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + " ORDER BY age LIMIT 1"; - - SearchHit[] hits = query(query).getHits(); - checkSuccessfulFieldCast(hits, "cast_age", "DOUBLE"); - for (int i = 0; i < hits.length; ++i) { - Assert.assertThat(hits[i].getFields().get("cast_age").getValue(), is(20.0)); - } - } - - @Test - public void castIntFieldToDoubleWithAliasOrderByTest() throws IOException { - String query = "SELECT CAST(age AS DOUBLE) AS alias FROM " + TestsConstants.TEST_INDEX_ACCOUNT + - " ORDER BY alias DESC LIMIT 1"; - - SearchHit[] hits = query(query).getHits(); - checkSuccessfulFieldCast(hits, "alias", "DOUBLE"); - for (int i = 0; i < hits.length; ++i) { - Assert.assertThat(hits[i].getFields().get("alias").getValue(), is(40.0)); - } - - } - - @Test - public void castIntFieldToFloatWithoutAliasJdbcFormatGroupByTest() { - JSONObject response = executeJdbcRequest( - "SELECT CAST(balance AS FLOAT) FROM " + - TestsConstants.TEST_INDEX_ACCOUNT + " GROUP BY balance DESC LIMIT 5"); - - verifySchema(response, - schema("CAST(balance AS FLOAT)", null, "float")); - - verifyDataRows(response, - rows(22026), - rows(23285), - rows(36038), - rows(39063), - rows(45493)); - } - - @Test - public void castIntFieldToFloatWithAliasJdbcFormatGroupByTest() { - JSONObject response = executeJdbcRequest( - "SELECT CAST(balance AS FLOAT) AS jdbc_float_alias " + - "FROM " + TestsConstants.TEST_INDEX_ACCOUNT + " GROUP BY jdbc_float_alias ASC LIMIT 5"); - - verifySchema(response, - schema("jdbc_float_alias", "jdbc_float_alias", "float")); - - verifyDataRows(response, - rows("22026.0"), - rows("23285.0"), - rows("36038.0"), - rows("39063.0"), - rows("45493.0")); - } - - @Test - public void castIntFieldToDoubleWithAliasJdbcFormatGroupByTest() { - JSONObject response = executeJdbcRequest( - "SELECT CAST(age AS DOUBLE) AS jdbc_double_alias " + - "FROM " + TestsConstants.TEST_INDEX_ACCOUNT + " GROUP BY jdbc_double_alias DESC LIMIT 5"); - - verifySchema(response, - schema("jdbc_double_alias", "jdbc_double_alias", "double")); - - verifyDataRows(response, - rows("31.0"), - rows("39.0"), - rows("26.0"), - rows("32.0"), - rows("35.0")); - } - - @Test - public void castKeywordFieldToDatetimeWithoutAliasJdbcFormatTest() { - JSONObject response = executeJdbcRequest("SELECT CAST(date_keyword AS DATETIME) FROM " - + TestsConstants.TEST_INDEX_DATE + " ORDER BY date_keyword"); - - verifySchema(response, schema("cast_date_keyword", null, "date")); - - verifyDataRows(response, - rows("2014-08-19 07:09:13.434"), - rows("2019-09-25 02:04:13.469")); - } - - @Test - public void castKeywordFieldToDatetimeWithAliasJdbcFormatTest() { - JSONObject response = executeJdbcRequest("SELECT CAST(date_keyword AS DATETIME) AS test_alias FROM " - + TestsConstants.TEST_INDEX_DATE + " ORDER BY date_keyword"); - - verifySchema(response, schema("test_alias", null, "date")); - - verifyDataRows(response, - rows("2014-08-19 07:09:13.434"), - rows("2019-09-25 02:04:13.469")); - } - - @Test - public void castFieldToDatetimeWithWhereClauseJdbcFormatTest() { - JSONObject response = executeJdbcRequest("SELECT CAST(date_keyword AS DATETIME) FROM " - + TestsConstants.TEST_INDEX_DATE + " WHERE date_keyword IS NOT NULL ORDER BY date_keyword"); - - verifySchema(response, schema("cast_date_keyword", null, "date")); - - verifyDataRows(response, - rows("2014-08-19 07:09:13.434"), - rows("2019-09-25 02:04:13.469")); - } - - @Test - public void castFieldToDatetimeWithGroupByJdbcFormatTest() { - JSONObject response = executeJdbcRequest("SELECT CAST(date_keyword AS DATETIME) AS test_alias FROM " - + TestsConstants.TEST_INDEX_DATE + " GROUP BY test_alias DESC"); - - verifySchema(response, schema("test_alias", "test_alias", "double")); - - verifyDataRows(response, - rows("2014-08-19T07:09:13.434Z"), - rows("2019-09-25T02:04:13.469Z")); - } - - @Test - public void castStatementInWhereClauseGreaterThanTest() { - JSONObject response = executeJdbcRequest("SELECT balance FROM " + TEST_INDEX_ACCOUNT - + " WHERE (account_number < CAST(age AS DOUBLE)) ORDER BY balance LIMIT 5"); - - verifySchema(response, schema("balance", null, "long")); - - verifyDataRows(response, - rows(4180), - rows(5686), - rows(7004), - rows(7831), - rows(14127)); - } - - @Test - public void castStatementInWhereClauseLessThanTest() { - JSONObject response = executeJdbcRequest("SELECT balance FROM " + TEST_INDEX_ACCOUNT - + " WHERE (account_number > CAST(age AS DOUBLE)) ORDER BY balance LIMIT 5"); - - verifySchema(response, schema("balance", null, "long")); - - verifyDataRows(response, - rows(1011), - rows(1031), - rows(1110), - rows(1133), - rows(1172)); - } - - @Test - public void castStatementInWhereClauseEqualToConstantTest() { - JSONObject response = executeJdbcRequest("SELECT balance FROM " + TEST_INDEX_ACCOUNT - + " WHERE (CAST(age AS DOUBLE) = 36.0) ORDER BY balance LIMIT 5"); - - verifySchema(response, schema("balance", null, "long")); - verifyDataRows(response, - rows(1249), - rows(1463), - rows(3960), - rows(5686), - rows(6025)); - } - - @Test - public void castStatementInWhereClauseLessThanConstantTest() { - JSONObject response = executeJdbcRequest("SELECT balance FROM " + TEST_INDEX_ACCOUNT - + " WHERE (CAST(age AS DOUBLE) < 36.0) ORDER BY balance LIMIT 5"); - - verifySchema(response, schema("balance", null, "long")); - - verifyDataRows(response, - rows(1011), - rows(1031), - rows(1110), - rows(1133), - rows(1172)); - } - - /** - * Testing compilation - * Result comparison is empty -> comparing different types (Date and keyword) - */ - @Test - public void castStatementInWhereClauseDatetimeCastTest() { - JSONObject response = executeJdbcRequest("SELECT date_keyword FROM " - + TestsConstants.TEST_INDEX_DATE - + " WHERE (CAST(date_keyword AS DATETIME) = \'2014-08-19T07:09:13.434Z\')"); - - String schema_result = "{\"name\":\"date_keyword\",\"type\":\"keyword\"}"; - assertEquals(response.getJSONArray("schema").get(0).toString(), schema_result); - } - - @Test - public void concat_ws_field_and_string() throws Exception { - //here is a bug,csv field with spa - String query = "SELECT " + - " concat_ws('-',age,'-') as age,address from " + - TEST_INDEX_ACCOUNT + " " + - " limit 10 "; - - assertThat( - executeQuery(query), - hitAny(kvString("/fields/age/0", endsWith("--"))) - ); - } - - /** - * Ignore this test case because painless doesn't whitelist String.split function. - * @see https://www.elastic.co/guide/en/elasticsearch/painless/7.0/painless-api-reference.html - */ - @Ignore - public void whereConditionLeftFunctionRightVariableEqualTest() throws Exception { - - String query = "SELECT " + - " * from " + - TestsConstants.TEST_INDEX + " " + - " where split(address,' ')[0]='806' limit 1000 "; - - assertThat(executeQuery(query).query("/hits/total"), equalTo(4)); - } - - /** - * Ignore this test case because painless doesn't whitelist String.split function. - * @see https://www.elastic.co/guide/en/elasticsearch/painless/7.0/painless-api-reference.html - */ - @Ignore - public void whereConditionLeftFunctionRightVariableGreatTest() throws Exception { - - String query = "SELECT " + - " * from " + - TestsConstants.TEST_INDEX + " " + - " where floor(split(address,' ')[0]+0) > 805 limit 1000 "; - - assertThat(executeQuery(query).query("/hits/total"), equalTo(223)); - } - - @Test - public void concat_ws_fields() throws Exception { - - //here is a bug,csv field with spa - String query = "SELECT " + - " concat_ws('-',age,address) as combine,address from " + - TEST_INDEX_ACCOUNT + " " + - " limit 10 "; - assertThat( - executeQuery(query), - hitAny(kvString("/fields/combine/0", containsString("-"))) - ); - } - - @Test - public void functionLogs() throws Exception { - String query = "SELECT log10(100) as a, log(1) as b, log(2, 4) as c, log2(8) as d from " - + TEST_INDEX_ACCOUNT + " limit 1"; - - assertThat( - executeQuery(query), - hitAny(both(kvDouble("/fields/a/0", equalTo(Math.log10(100)))) - .and(kvDouble("/fields/b/0", equalTo(Math.log(1)))) - .and(kvDouble("/fields/c/0", closeTo(Math.log(4)/Math.log(2), 0.0001))) - .and(kvDouble("/fields/d/0", closeTo(Math.log(8)/Math.log(2), 0.0001)))) - ); - } - - @Test - public void functionPow() throws Exception { - String query = "SELECT pow(account_number, 2) as key,"+ - "abs(age - 60) as new_age from " + TEST_INDEX_ACCOUNT + " WHERE firstname = 'Virginia' and lastname='Ayala' limit 1"; - - assertThat( - executeQuery(query), - hitAny(both(kvDouble("/fields/new_age/0", equalTo(21.0))).and(kvDouble("/fields/key/0", equalTo(625.0)))) - ); - } - - @Test - public void operatorSubstring() throws IOException { - assertThat( - executeQuery("SELECT substring('sampleName', 1, 4) AS substring FROM " + TEST_INDEX_ACCOUNT), - hitAny(kvString("/fields/substring/0", equalTo("samp"))) - ); - - assertThat( - executeQuery("SELECT substring('sampleName', 0, 20) AS substring FROM " + TEST_INDEX_ACCOUNT), - hitAny(kvString("/fields/substring/0", equalTo("sampleName"))) - ); - } - - @Test - public void operatorLength() throws IOException { - assertThat( - executeQuery("SELECT LENGTH(lastname) FROM " + TEST_INDEX_ACCOUNT - + " WHERE lastname IS NOT NULL GROUP BY LENGTH(lastname) ORDER BY LENGTH(lastname)", "jdbc"), - containsString("\"type\": \"integer\"") - ); - - assertThat( - executeQuery("SELECT LENGTH('sampleName') AS length FROM " + TEST_INDEX_ACCOUNT), - hitAny(kvInt("/fields/length/0", equalTo(10))) - ); - - } - - @Test - public void operatorReplace() { - String query = "SELECT REPLACE('elastic', 'el', 'fant') FROM " + TEST_INDEX_ACCOUNT; - assertThat( - executeQuery(query, "jdbc"), - containsString("fantastic") - ); - } - - @Test - public void operatorLocate() throws IOException { - String query = "SELECT LOCATE('a', lastname, 0) FROM " + TEST_INDEX_ACCOUNT - + " WHERE lastname IS NOT NULL GROUP BY LOCATE('a', lastname, 0) ORDER BY LOCATE('a', lastname, 0)"; - assertThat( - executeQuery(query, "jdbc"), containsString("\"type\": \"integer\"") - ); - - assertThat( - executeQuery("SELECT LOCATE('a', 'sampleName', 3) AS locate FROM " + TEST_INDEX_ACCOUNT), - hitAny(kvInt("/fields/locate/0", equalTo(8))) - ); - assertThat( - executeQuery("SELECT LOCATE('a', 'sampleName') AS locate FROM " + TEST_INDEX_ACCOUNT), - hitAny(kvInt("/fields/locate/0", equalTo(2))) - ); - } - - @Test - public void rtrim() throws IOException { - assertThat( - executeQuery("SELECT RTRIM(' sampleName ') AS rtrim FROM " + TEST_INDEX_ACCOUNT), - hitAny(kvString("/fields/rtrim/0", equalTo(" sampleName"))) - ); - } - - @Test - public void ltrim() throws IOException { - assertThat( - executeQuery("SELECT LTRIM(' sampleName ') AS ltrim FROM " + TEST_INDEX_ACCOUNT), - hitAny(kvString("/fields/ltrim/0",equalTo( "sampleName "))) - ); - } - - @Test - public void ascii() throws IOException { - assertThat( - executeQuery("SELECT ASCII(lastname) FROM " + TEST_INDEX_ACCOUNT - + " WHERE lastname IS NOT NULL GROUP BY ASCII(lastname) ORDER BY ASCII(lastname) LIMIT 5", - "jdbc"), - containsString("\"type\": \"integer\"") - ); - assertThat( - executeQuery("SELECT ASCII('sampleName') AS ascii FROM " + TEST_INDEX_ACCOUNT), - hitAny(kvInt("/fields/ascii/0", equalTo(115))) - ); - } - - /** - * The following tests for LEFT and RIGHT are ignored because the ES client fails to parse "LEFT"/"RIGHT" in - * the integTest - */ - @Ignore - @Test - public void left() throws IOException { - assertThat( - executeQuery("SELECT LEFT('sample', 2) AS left FROM " + TEST_INDEX_ACCOUNT + " ORDER BY left"), - hitAny(kvString("/fields/left/0", equalTo("sa"))) - ); - assertThat( - executeQuery("SELECT LEFT('sample', 20) AS left FROM " + TEST_INDEX_ACCOUNT + " ORDER BY left"), - hitAny(kvString("/fields/left/0", equalTo("sample"))) - ); - } - - @Ignore - @Test - public void right() throws IOException { - assertThat( - executeQuery("SELECT RIGHT('elastic', 3) AS right FROM " + TEST_INDEX_ACCOUNT + " ORDER BY right"), - hitAny(kvString("/fields/right/0", equalTo("tic"))) - ); - assertThat( - executeQuery("SELECT RIGHT('elastic', 20) AS right FROM " + TEST_INDEX_ACCOUNT + " ORDER BY right"), - hitAny(kvString("/fields/right/0", equalTo("elastic"))) - ); - } - - @Test - public void ifFuncShouldPassJDBC() { - JSONObject response = executeJdbcRequest( - "SELECT IF(age > 30, 'True', 'False') AS Ages FROM " + TEST_INDEX_ACCOUNT - + " WHERE age IS NOT NULL GROUP BY Ages"); - assertEquals("Ages", response.query("/schema/0/name")); - assertEquals("Ages", response.query("/schema/0/alias")); - assertEquals("double", response.query("/schema/0/type")); - } - - @Test - public void ifFuncWithBinaryComparisonAsConditionTest() throws IOException { - assertThat( - executeQuery("SELECT IF(2 > 0, 'hello', 'world') AS ifTrue FROM " + TEST_INDEX_ACCOUNT), - hitAny(kvString("/fields/ifTrue/0", equalTo("hello"))) - ); - assertThat( - executeQuery("SELECT IF(2 = 0, 'hello', 'world') AS ifFalse FROM " + TEST_INDEX_ACCOUNT), - hitAny(kvString("/fields/ifFalse/0", equalTo("world"))) - ); - } - - @Test - public void ifFuncWithBooleanExprInputAsConditionTest() throws IOException { - assertThat( - executeQuery("SELECT IF(true, 1, 0) AS ifBoolean FROM " + TEST_INDEX_ACCOUNT), - hitAny(kvInt("/fields/ifBoolean/0", equalTo(1))) - ); - } - - @Test - public void ifFuncWithNullInputAsConditionTest() throws IOException { - assertThat( - executeQuery("SELECT IF(null, 1, 0) AS ifNull FROM " + TEST_INDEX_ACCOUNT), - hitAny(kvInt("/fields/ifNull/0", equalTo(0))) - ); - } - - @Test - public void ifnullShouldPassJDBC() throws IOException { - JSONObject response = executeJdbcRequest( - "SELECT IFNULL(lastname, 'unknown') AS name FROM " + TEST_INDEX_ACCOUNT - + " GROUP BY name"); - assertEquals("name", response.query("/schema/0/name")); - assertEquals("name", response.query("/schema/0/alias")); - assertEquals("double", response.query("/schema/0/type")); - } - - @Test - public void ifnullWithNotNullInputTest() throws IOException { - assertThat( - executeQuery("SELECT IFNULL('sample', 'IsNull') AS ifnull FROM " + TEST_INDEX_ACCOUNT), - hitAny(kvString("/fields/ifnull/0", equalTo("sample"))) - ); - } - - @Test - public void ifnullWithNullInputTest() throws IOException { - assertThat( - executeQuery("SELECT IFNULL(null, 10) AS ifnull FROM " + TEST_INDEX_ACCOUNT), - hitAny(kvInt("/fields/ifnull/0", equalTo(10))) - ); - assertThat( - executeQuery("SELECT IFNULL('', 10) AS ifnull FROM " + TEST_INDEX_ACCOUNT), - hitAny(kvString("/fields/ifnull/0", equalTo(""))) - ); - } - - @Test - public void isnullShouldPassJDBC() { - JSONObject response = - executeJdbcRequest("SELECT ISNULL(lastname) AS name FROM " + TEST_INDEX_ACCOUNT + " GROUP BY name"); - assertEquals("name", response.query("/schema/0/name")); - assertEquals("name", response.query("/schema/0/alias")); - assertEquals("integer", response.query("/schema/0/type")); - } - - @Test - public void isnullWithNotNullInputTest() throws IOException { - assertThat( - executeQuery("SELECT ISNULL('elastic') AS isnull FROM " + TEST_INDEX_ACCOUNT), - hitAny(kvInt("/fields/isnull/0", equalTo(0))) - ); - assertThat( - executeQuery("SELECT ISNULL('') AS isnull FROM " + TEST_INDEX_ACCOUNT), - hitAny(kvInt("/fields/isnull/0", equalTo(0))) - ); - } - - @Test - public void isnullWithNullInputTest() throws IOException { - assertThat( - executeQuery("SELECT ISNULL(null) AS isnull FROM " + TEST_INDEX_ACCOUNT), - hitAny(kvInt("/fields/isnull/0", equalTo(1))) - ); - } - - @Test - public void isnullWithMathExpr() throws IOException{ - assertThat( - executeQuery("SELECT ISNULL(1+1) AS isnull FROM " + TEST_INDEX_ACCOUNT), - hitAny(kvInt("/fields/isnull/0", equalTo(0))) - ); - assertThat( - executeQuery("SELECT ISNULL(1+1*1/0) AS isnull FROM " + TEST_INDEX_ACCOUNT), - hitAny(kvInt("/fields/isnull/0", equalTo(1))) - ); - } - - /** - * Ignore this test case because painless doesn't whitelist String.split function. - * @see https://www.elastic.co/guide/en/elasticsearch/painless/7.0/painless-api-reference.html - */ - @Ignore - public void split_field() throws Exception { - - //here is a bug,csv field with spa - String query = "SELECT " + - " split(address,' ')[0],age from " + - TestsConstants.TEST_INDEX + " where address is not null " + - " limit 10 "; - } - - @Test - public void literal() throws Exception { - String query = "SELECT 10 "+ - "from " + TEST_INDEX_ACCOUNT + " limit 1"; - final SearchHit[] hits = query(query).getHits(); - assertThat(hits[0].getFields(), hasValue(contains(10))); - } - - @Test - public void literalWithDoubleValue() throws Exception { - String query = "SELECT 10.0 "+ - "from " + TEST_INDEX_ACCOUNT + " limit 1"; - - final SearchHit[] hits = query(query).getHits(); - assertThat(hits[0].getFields(), hasValue(contains(10.0))); - } - - @Test - public void literalWithAlias() throws Exception { - String query = "SELECT 10 as key "+ - "from " + TEST_INDEX_ACCOUNT + " limit 1"; - final SearchHit[] hits = query(query).getHits(); - - assertThat(hits.length, is(1)); - assertThat(hits[0].getFields(), hasEntry(is("key"), contains(10))); - } - - @Test - public void literalMultiField() throws Exception { - String query = "SELECT 1, 2 "+ - "from " + TEST_INDEX_ACCOUNT + " limit 1"; - final SearchHit[] hits = query(query).getHits(); - - assertThat(hits.length, is(1)); - assertThat(hits[0].getFields(), allOf(hasValue(contains(1)), hasValue(contains(2)))); - } - - private SearchHits query(String query) throws IOException { - final String rsp = executeQueryWithStringOutput(query); - - final XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser( - NamedXContentRegistry.EMPTY, - LoggingDeprecationHandler.INSTANCE, - rsp); - return SearchResponse.fromXContent(parser).getHits(); - } - - private void checkSuccessfulFieldCast(SearchHit[] hits, String field, String castType) { - for (int i = 0; i < hits.length; ++i) { - Assert.assertThat(hits[i].getFields(), IsMapContaining.hasKey(field)); - switch (castType) { - case "FLOAT": - assertTrue(hits[i].getFields().get(field).getValue() instanceof Float); - break; - case "DOUBLE": - assertTrue(hits[i].getFields().get(field).getValue() instanceof Double); - break; - case "INT": - assertTrue(hits[i].getFields().get(field).getValue() instanceof Integer); - break; - case "STRING": - assertTrue(hits[i].getFields().get(field).getValue() instanceof String); - break; - case "DATETIME": - assertTrue(hits[i].getFields().get(field).getValue() instanceof Date); - break; - case "LONG": - assertTrue(hits[i].getFields().get(field).getValue() instanceof Long); - break; - } - } - } - - private JSONObject executeJdbcRequest(String query) { - return new JSONObject(executeQuery(query, "jdbc")); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java deleted file mode 100644 index 30c3e709f2..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java +++ /dev/null @@ -1,537 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import com.google.common.base.Strings; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.Response; -import org.json.JSONArray; -import org.json.JSONObject; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; - -import javax.management.MBeanServerInvocationHandler; -import javax.management.ObjectName; -import javax.management.remote.JMXConnector; -import javax.management.remote.JMXConnectorFactory; -import javax.management.remote.JMXServiceURL; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Locale; - -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.createIndexByRestClient; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getAccountIndexMapping; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getBankIndexMapping; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getBankWithNullValuesIndexMapping; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getDateIndexMapping; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getDateTimeIndexMapping; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getNestedSimpleIndexMapping; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getDogIndexMapping; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getDogs2IndexMapping; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getDogs3IndexMapping; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getEmployeeNestedTypeIndexMapping; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getGameOfThronesIndexMapping; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getJoinTypeIndexMapping; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getLocationIndexMapping; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getNestedTypeIndexMapping; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getOdbcIndexMapping; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getOrderIndexMapping; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getPeople2IndexMapping; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getPhraseIndexMapping; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getResponseBody; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getWeblogsIndexMapping; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.isIndexExist; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.loadDataByRestClient; -import static com.amazon.opendistroforelasticsearch.sql.plugin.RestSqlAction.EXPLAIN_API_ENDPOINT; -import static com.amazon.opendistroforelasticsearch.sql.plugin.RestSqlAction.QUERY_API_ENDPOINT; -import static com.amazon.opendistroforelasticsearch.sql.plugin.RestSqlAction.CURSOR_CLOSE_ENDPOINT; - -/** - * SQL plugin integration test base class. - * - * The execution of order is as follows: - * - * ESRestTestCase: 1) initClient() N+1) closeClient() - * \ / - * SQLIntegTestCase: 2) setUpIndices() -> 4) setUpIndices() ... -> N) cleanUpIndices() - * \ \ - * XXXTIT: 3) init() 5) init() - */ -public abstract class SQLIntegTestCase extends ODFERestTestCase { - - public static final String PERSISTENT = "persistent"; - public static final String TRANSIENT = "transient"; - - @Before - public void setUpIndices() throws Exception { - if (client() == null) { - initClient(); - } - - increaseScriptMaxCompilationsRate(); - init(); - } - - @Override - protected boolean preserveClusterUponCompletion() { - return true; // Preserve test index, template and settings between test cases - } - - /** - * We need to be able to dump the jacoco coverage before cluster is shut down. - * The new internal testing framework removed some of the gradle tasks we were listening to - * to choose a good time to do it. This will dump the executionData to file after each test. - * TODO: This is also currently just overwriting integTest.exec with the updated execData without - * resetting after writing each time. This can be improved to either write an exec file per test - * or by letting jacoco append to the file - */ - public interface IProxy { - byte[] getExecutionData(boolean reset); - void dump(boolean reset); - void reset(); - } - - @AfterClass - public static void dumpCoverage() { - // jacoco.dir is set in sqlplugin-coverage.gradle, if it doesn't exist we don't - // want to collect coverage so we can return early - String jacocoBuildPath = System.getProperty("jacoco.dir"); - if (Strings.isNullOrEmpty(jacocoBuildPath)) { - return; - } - - String serverUrl = "service:jmx:rmi:///jndi/rmi://127.0.0.1:7777/jmxrmi"; - try(JMXConnector connector = JMXConnectorFactory.connect(new JMXServiceURL(serverUrl))) { - IProxy proxy = MBeanServerInvocationHandler.newProxyInstance( - connector.getMBeanServerConnection(), new ObjectName("org.jacoco:type=Runtime"), IProxy.class, - false); - - Path path = Paths.get(jacocoBuildPath + "/integTest.exec"); - Files.write(path, proxy.getExecutionData(false)); - } catch (Exception ex) { - throw new RuntimeException("Failed to dump coverage", ex); - } - } - - /** - * As JUnit JavaDoc says: - * "The @AfterClass methods declared in superclasses will be run after those of the current class." - * So this method is supposed to run before closeClients() in parent class. - */ - @AfterClass - public static void cleanUpIndices() throws IOException { - wipeAllODFEIndices(); - wipeAllClusterSettings(); - } - - /** - * Increase script.max_compilations_rate to large enough, which is only 75/5min by default. - * This issue is due to our painless script not using params passed to compiled script. - */ - private void increaseScriptMaxCompilationsRate() throws IOException { - updateClusterSettings(new ClusterSetting("transient", "script.max_compilations_rate", "10000/1m")); - } - - protected static void wipeAllClusterSettings() throws IOException { - updateClusterSettings(new ClusterSetting("persistent", "*", null)); - updateClusterSettings(new ClusterSetting("transient", "*", null)); - } - - /** - * Provide for each test to load test index, data and other setup work - */ - protected void init() throws Exception {} - - /** - * Make it thread-safe in case tests are running in parallel but does not guarantee - * if test like DeleteIT that mutates cluster running in parallel. - */ - protected synchronized void loadIndex(Index index) throws IOException { - String indexName = index.getName(); - String mapping = index.getMapping(); - String dataSet = index.getDataSet(); - - if (!isIndexExist(client(), indexName)) { - createIndexByRestClient(client(), indexName, mapping); - loadDataByRestClient(client(), indexName, dataSet); - } - } - - protected Request getSqlRequest(String request, boolean explain) { - return getSqlRequest(request, explain, "json"); - } - - protected Request getSqlRequest(String request, boolean explain, String requestType) { - String queryEndpoint = String.format("%s?format=%s", QUERY_API_ENDPOINT, requestType); - Request sqlRequest = new Request("POST", explain ? EXPLAIN_API_ENDPOINT : queryEndpoint); - sqlRequest.setJsonEntity(request); - RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); - restOptionsBuilder.addHeader("Content-Type", "application/json"); - sqlRequest.setOptions(restOptionsBuilder); - - return sqlRequest; - } - - protected Request getSqlCursorCloseRequest(String cursorRequest) { - String queryEndpoint = String.format("%s?format=%s", CURSOR_CLOSE_ENDPOINT, "jdbc"); - Request sqlRequest = new Request("POST", queryEndpoint); - sqlRequest.setJsonEntity(cursorRequest); - RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); - restOptionsBuilder.addHeader("Content-Type", "application/json"); - sqlRequest.setOptions(restOptionsBuilder); - - return sqlRequest; - } - - protected String executeQuery(String query, String requestType) { - try { - String endpoint = "/_opendistro/_sql?format=" + requestType; - String requestBody = makeRequest(query); - - Request sqlRequest = new Request("POST", endpoint); - sqlRequest.setJsonEntity(requestBody); - - Response response = client().performRequest(sqlRequest); - Assert.assertEquals(200, response.getStatusLine().getStatusCode()); - String responseString = getResponseBody(response, true); - - return responseString; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - protected String executeFetchQuery(String query, int fetchSize, String requestType) throws IOException { - String endpoint = "/_opendistro/_sql?format=" + requestType; - String requestBody = makeRequest(query, fetchSize); - - Request sqlRequest = new Request("POST", endpoint); - sqlRequest.setJsonEntity(requestBody); - - Response response = client().performRequest(sqlRequest); - String responseString = getResponseBody(response, true); - return responseString; - } - - protected String executeFetchLessQuery(String query, String requestType) throws IOException { - - String endpoint = "/_opendistro/_sql?format=" + requestType; - String requestBody = makeFetchLessRequest(query); - - Request sqlRequest = new Request("POST", endpoint); - sqlRequest.setJsonEntity(requestBody); - - Response response = client().performRequest(sqlRequest); - String responseString = getResponseBody(response, true); - return responseString; - } - - protected Request buildGetEndpointRequest(final String sqlQuery) { - - final String utf8CharsetName = StandardCharsets.UTF_8.name(); - String urlEncodedQuery = ""; - - try { - urlEncodedQuery = URLEncoder.encode(sqlQuery, utf8CharsetName); - } catch (UnsupportedEncodingException e) { - // Will never reach here since UTF-8 is always supported - Assert.fail(utf8CharsetName + " not available"); - } - - final String requestUrl = String.format(Locale.ROOT, "%s?sql=%s&format=%s", QUERY_API_ENDPOINT, - urlEncodedQuery, "json"); - return new Request("GET", requestUrl); - } - - protected JSONObject executeQuery(final String sqlQuery) throws IOException { - - final String requestBody = makeRequest(sqlQuery); - return executeRequest(requestBody); - } - - protected String explainQuery(final String sqlQuery) throws IOException { - - final String requestBody = makeRequest(sqlQuery); - return executeExplainRequest(requestBody); - } - - protected String executeQueryWithStringOutput(final String sqlQuery) throws IOException { - - final String requestString = makeRequest(sqlQuery); - return executeRequest(requestString, false); - } - - protected JSONObject executeRequest(final String requestBody) throws IOException { - - return new JSONObject(executeRequest(requestBody, false)); - } - - protected String executeExplainRequest(final String requestBody) throws IOException { - - return executeRequest(requestBody, true); - } - - private String executeRequest(final String requestBody, final boolean isExplainQuery) throws IOException { - - Request sqlRequest = getSqlRequest(requestBody, isExplainQuery); - return executeRequest(sqlRequest); - } - - protected static String executeRequest(final Request request) throws IOException { - - Response response = client().performRequest(request); - Assert.assertEquals(200, response.getStatusLine().getStatusCode()); - return getResponseBody(response); - } - - protected JSONObject executeQueryWithGetRequest(final String sqlQuery) throws IOException { - - final Request request = buildGetEndpointRequest(sqlQuery); - final String result = executeRequest(request); - return new JSONObject(result); - } - - protected JSONObject executeCursorQuery(final String cursor) throws IOException { - final String requestBody = makeCursorRequest(cursor); - Request sqlRequest = getSqlRequest(requestBody, false, "jdbc"); - return new JSONObject(executeRequest(sqlRequest)); - } - - protected JSONObject executeCursorCloseQuery(final String cursor) throws IOException { - final String requestBody = makeCursorRequest(cursor); - Request sqlRequest = getSqlCursorCloseRequest(requestBody); - return new JSONObject(executeRequest(sqlRequest)); - } - - protected static JSONObject updateClusterSettings(ClusterSetting setting) throws IOException { - Request request = new Request("PUT", "/_cluster/settings"); - String persistentSetting = String.format(Locale.ROOT, - "{\"%s\": {\"%s\": %s}}", setting.type, setting.name, setting.value); - request.setJsonEntity(persistentSetting); - RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); - restOptionsBuilder.addHeader("Content-Type", "application/json"); - request.setOptions(restOptionsBuilder); - return new JSONObject(executeRequest(request)); - } - - protected static JSONObject getAllClusterSettings() throws IOException { - Request request = new Request("GET", "/_cluster/settings?flat_settings&include_defaults"); - RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); - restOptionsBuilder.addHeader("Content-Type", "application/json"); - request.setOptions(restOptionsBuilder); - return new JSONObject(executeRequest(request)); - } - - protected static class ClusterSetting { - private final String type; - private final String name; - private final String value; - - ClusterSetting(String type, String name, String value) { - this.type = type; - this.name = name; - this.value = (value == null) ? "null" : ("\"" + value + "\""); - } - - ClusterSetting nullify() { - return new ClusterSetting(type, name, null); - } - - @Override - public String toString() { - return "ClusterSetting{" + - "type='" + type + '\'' + - ", path='" + name + '\'' + - ", value='" + value + '\'' + - '}'; - } - } - - protected String makeRequest(String query) { - return makeRequest(query, 0); - } - - protected String makeRequest(String query, int fetch_size) { - return String.format("{\n" + - " \"fetch_size\": \"%s\",\n" + - " \"query\": \"%s\"\n" + - "}", fetch_size, query); - } - - protected String makeFetchLessRequest(String query) { - return String.format("{\n" + - " \"query\": \"%s\"\n" + - "}", query); - } - - protected String makeCursorRequest(String cursor) { - return String.format("{\"cursor\":\"%s\"}" , cursor); - } - - protected JSONArray getHits(JSONObject response) { - Assert.assertTrue(response.getJSONObject("hits").has("hits")); - - return response.getJSONObject("hits").getJSONArray("hits"); - } - - protected int getTotalHits(JSONObject response) { - Assert.assertTrue(response.getJSONObject("hits").has("total")); - Assert.assertTrue(response.getJSONObject("hits").getJSONObject("total").has("value")); - - return response.getJSONObject("hits").getJSONObject("total").getInt("value"); - } - - protected JSONObject getSource(JSONObject hit) { - return hit.getJSONObject("_source"); - } - - /** - * Enum for associating test index with relevant mapping and data. - */ - public enum Index { - ONLINE(TestsConstants.TEST_INDEX_ONLINE, - "online", - null, - "src/test/resources/online.json"), - ACCOUNT(TestsConstants.TEST_INDEX_ACCOUNT, - "account", - getAccountIndexMapping(), - "src/test/resources/accounts.json"), - PHRASE(TestsConstants.TEST_INDEX_PHRASE, - "phrase", - getPhraseIndexMapping(), - "src/test/resources/phrases.json"), - DOG(TestsConstants.TEST_INDEX_DOG, - "dog", - getDogIndexMapping(), - "src/test/resources/dogs.json"), - DOGS2(TestsConstants.TEST_INDEX_DOG2, - "dog", - getDogs2IndexMapping(), - "src/test/resources/dogs2.json"), - DOGS3(TestsConstants.TEST_INDEX_DOG3, - "dog", - getDogs3IndexMapping(), - "src/test/resources/dogs3.json"), - DOGSSUBQUERY(TestsConstants.TEST_INDEX_DOGSUBQUERY, - "dog", - getDogIndexMapping(), - "src/test/resources/dogsubquery.json"), - PEOPLE(TestsConstants.TEST_INDEX_PEOPLE, - "people", - null, - "src/test/resources/peoples.json"), - PEOPLE2(TestsConstants.TEST_INDEX_PEOPLE2, - "people", - getPeople2IndexMapping(), - "src/test/resources/people2.json"), - GAME_OF_THRONES(TestsConstants.TEST_INDEX_GAME_OF_THRONES, - "gotCharacters", - getGameOfThronesIndexMapping(), - "src/test/resources/game_of_thrones_complex.json"), - SYSTEM(TestsConstants.TEST_INDEX_SYSTEM, - "systems", - null, - "src/test/resources/systems.json"), - ODBC(TestsConstants.TEST_INDEX_ODBC, - "odbc", - getOdbcIndexMapping(), - "src/test/resources/odbc-date-formats.json"), - LOCATION(TestsConstants.TEST_INDEX_LOCATION, - "location", - getLocationIndexMapping(), - "src/test/resources/locations.json"), - LOCATION_TWO(TestsConstants.TEST_INDEX_LOCATION2, - "location2", - getLocationIndexMapping(), - "src/test/resources/locations2.json"), - NESTED(TestsConstants.TEST_INDEX_NESTED_TYPE, - "nestedType", - getNestedTypeIndexMapping(), - "src/test/resources/nested_objects.json"), - NESTED_WITH_QUOTES(TestsConstants.TEST_INDEX_NESTED_WITH_QUOTES, - "nestedType", - getNestedTypeIndexMapping(), - "src/test/resources/nested_objects_quotes_in_values.json"), - EMPLOYEE_NESTED(TestsConstants.TEST_INDEX_EMPLOYEE_NESTED, - "_doc", - getEmployeeNestedTypeIndexMapping(), - "src/test/resources/employee_nested.json"), - JOIN(TestsConstants.TEST_INDEX_JOIN_TYPE, - "joinType", - getJoinTypeIndexMapping(), - "src/test/resources/join_objects.json"), - BANK(TestsConstants.TEST_INDEX_BANK, - "account", - getBankIndexMapping(), - "src/test/resources/bank.json"), - BANK_TWO(TestsConstants.TEST_INDEX_BANK_TWO, - "account_two", - getBankIndexMapping(), - "src/test/resources/bank_two.json"), - BANK_WITH_NULL_VALUES(TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES, - "account_null", - getBankWithNullValuesIndexMapping(), - "src/test/resources/bank_with_null_values.json"), - ORDER(TestsConstants.TEST_INDEX_ORDER, - "_doc", - getOrderIndexMapping(), - "src/test/resources/order.json"), - WEBLOG(TestsConstants.TEST_INDEX_WEBLOG, - "weblog", - getWeblogsIndexMapping(), - "src/test/resources/weblogs.json"), - DATE(TestsConstants.TEST_INDEX_DATE, - "dates", - getDateIndexMapping(), - "src/test/resources/dates.json"), - DATETIME(TestsConstants.TEST_INDEX_DATE_TIME, - "_doc", - getDateTimeIndexMapping(), - "src/test/resources/datetime.json"), - NESTED_SIMPLE(TestsConstants.TEST_INDEX_NESTED_SIMPLE, - "_doc", - getNestedSimpleIndexMapping(), - "src/test/resources/nested_simple.json"); - - private final String name; - private final String type; - private final String mapping; - private final String dataSet; - - Index(String name, String type, String mapping, String dataSet) { - this.name = name; - this.type = type; - this.mapping = mapping; - this.dataSet = dataSet; - } - - public String getName() { return this.name; } - - public String getType() { return this.type; } - - public String getMapping() { return this.mapping; } - - public String getDataSet() { return this.dataSet; } - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ShowIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ShowIT.java deleted file mode 100644 index 9caeeb1c4b..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ShowIT.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import org.json.JSONObject; -import org.junit.Assert; -import org.junit.Test; - -import java.io.IOException; - -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.createIndexByRestClient; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.isIndexExist; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; - -public class ShowIT extends SQLIntegTestCase { - - @Override - protected void init() { - // Note: not using the existing TEST_INDEX_* indices, since underscore in the names causes issues - createEmptyIndexIfNotExist("abcdefg"); - createEmptyIndexIfNotExist("abcdefghijk"); - createEmptyIndexIfNotExist("abcdijk"); - } - - @Test - public void showAllMatchAll() throws IOException { - - showIndexTest("%", 3, false); - } - - @Test - public void showIndexMatchPrefix() throws IOException { - - showIndexTest("abcdefg" + "%", 2, true); - } - - @Test - public void showIndexMatchSuffix() throws IOException { - - showIndexTest("%ijk", 2, true); - } - - @Test - public void showIndexMatchExact() throws IOException { - - showIndexTest("abcdefg", 1, true); - } - - private void showIndexTest(String querySuffix, int expectedMatches, boolean exactMatch) throws IOException { - - final String query = "SHOW TABLES LIKE " + querySuffix; - JSONObject result = executeQuery(query); - - if (exactMatch) { - Assert.assertThat(result.length(), equalTo(expectedMatches)); - } else { - Assert.assertThat(result.length(), greaterThanOrEqualTo(expectedMatches)); - } - for (String indexName : result.keySet()) { - Assert.assertTrue(result.getJSONObject(indexName).has("mappings")); - } - } - - private void createEmptyIndexIfNotExist(String indexName) { - if (!isIndexExist(client(), indexName)) { - createIndexByRestClient(client(), indexName, null); - } - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SourceFieldIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SourceFieldIT.java deleted file mode 100644 index 5d819f9b96..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SourceFieldIT.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHits; -import org.json.JSONObject; -import org.junit.Assert; -import org.junit.Test; - -import java.io.IOException; -import java.util.Set; - -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_ACCOUNT; - -public class SourceFieldIT extends SQLIntegTestCase { - - @Override - protected void init() throws Exception { - loadIndex(Index.ACCOUNT); - } - - @Test - public void includeTest() throws IOException { - SearchHits response = query(String.format("SELECT include('*name','*ge'),include('b*'),include('*ddre*'),include('gender') FROM %s/account LIMIT 1000", TEST_INDEX_ACCOUNT)); - for (SearchHit hit : response.getHits()) { - Set keySet = hit.getSourceAsMap().keySet(); - for (String field : keySet) { - Assert.assertTrue(field.endsWith("name") || field.endsWith("ge") || field.startsWith("b") || field.contains("ddre") || field.equals("gender")); - } - } - - } - - @Test - public void excludeTest() throws IOException { - - SearchHits response = query(String.format("SELECT exclude('*name','*ge'),exclude('b*'),exclude('*ddre*'),exclude('gender') FROM %s/account LIMIT 1000", TEST_INDEX_ACCOUNT)); - - for (SearchHit hit : response.getHits()) { - Set keySet = hit.getSourceAsMap().keySet(); - for (String field : keySet) { - Assert.assertFalse(field.endsWith("name") || field.endsWith("ge") || field.startsWith("b") || field.contains("ddre") || field.equals("gender")); - } - } - } - - @Test - public void allTest() throws IOException { - - SearchHits response = query(String.format("SELECT exclude('*name','*ge'),include('b*'),exclude('*ddre*'),include('gender') FROM %s/account LIMIT 1000", TEST_INDEX_ACCOUNT)); - - for (SearchHit hit : response.getHits()) { - Set keySet = hit.getSourceAsMap().keySet(); - for (String field : keySet) { - Assert.assertFalse(field.endsWith("name") || field.endsWith("ge") || field.contains("ddre") ); - Assert.assertTrue(field.startsWith("b") || field.equals("gender")); - } - } - } - - private SearchHits query(String query) throws IOException { - final JSONObject jsonObject = executeQuery(query); - - final XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser( - NamedXContentRegistry.EMPTY, - LoggingDeprecationHandler.INSTANCE, - jsonObject.toString()); - return SearchResponse.fromXContent(parser).getHits(); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SubqueryIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SubqueryIT.java deleted file mode 100644 index b006b7a229..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SubqueryIT.java +++ /dev/null @@ -1,382 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; -import com.google.common.collect.Ordering; -import org.elasticsearch.client.ResponseException; -import org.json.JSONArray; -import org.json.JSONObject; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; - -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_ACCOUNT; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_DOGSUBQUERY; -import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_EMPLOYEE_NESTED; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.hitAll; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.kvInt; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.kvString; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.rows; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.schema; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyDataRows; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifySchema; -import static org.hamcrest.Matchers.both; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.core.Is.is; - -public class SubqueryIT extends SQLIntegTestCase { - - @Rule - public ExpectedException exceptionRule = ExpectedException.none(); - - - @Override - protected void init() throws Exception { - loadIndex(Index.ACCOUNT); - loadIndex(Index.DOGSSUBQUERY); - loadIndex(Index.EMPLOYEE_NESTED); - } - - @Test - public void testIN() throws IOException { - String query = String.format(Locale.ROOT, - "SELECT dog_name " + - "FROM %s A " + - "WHERE holdersName IN (SELECT firstname FROM %s B) " + - "AND dog_name <> 'babala'", - TEST_INDEX_DOGSUBQUERY, TEST_INDEX_ACCOUNT); - - JSONObject response = executeQuery(query); - assertThat( - response, - hitAll( - kvString("/_source/A.dog_name", is("snoopy")), - kvString("/_source/A.dog_name", is("gogo")) - ) - ); - } - - @Test - public void testINWithAlias() throws IOException { - String query = String.format(Locale.ROOT, - "SELECT A.dog_name " + - "FROM %s A " + - "WHERE A.holdersName IN (SELECT B.firstname FROM %s B) " + - "AND A.dog_name <> 'babala'", - TEST_INDEX_DOGSUBQUERY, TEST_INDEX_ACCOUNT); - - JSONObject response = executeQuery(query); - assertThat( - response, - hitAll( - kvString("/_source/A.dog_name", is("snoopy")), - kvString("/_source/A.dog_name", is("gogo")) - ) - ); - } - - @Test - public void testINSelectAll() throws IOException { - String query = String.format(Locale.ROOT, - "SELECT * " + - "FROM %s A " + - "WHERE holdersName IN (SELECT firstname FROM %s B) " + - "AND dog_name <> 'babala'", - TEST_INDEX_DOGSUBQUERY, TEST_INDEX_ACCOUNT); - - JSONObject response = executeQuery(query); - assertThat( - response, - hitAll( - both(kvString("/_source/A.dog_name", is("snoopy"))) - .and(kvString("/_source/A.holdersName", is("Hattie"))) - .and(kvInt("/_source/A.age", is(4))), - both(kvString("/_source/A.dog_name", is("gogo"))) - .and(kvString("/_source/A.holdersName", is("Gabrielle"))) - .and(kvInt("/_source/A.age", is(6))) - ) - ); - } - - @Test - public void testINWithInnerWhere() throws IOException { - String query = String.format(Locale.ROOT, - "SELECT dog_name " + - "FROM %s A " + - "WHERE holdersName IN (SELECT firstname FROM %s B WHERE age <> 36) " + - "AND dog_name <> 'babala'", - TEST_INDEX_DOGSUBQUERY, TEST_INDEX_ACCOUNT); - - JSONObject response = executeQuery(query); - assertThat( - response, - hitAll( - kvString("/_source/A.dog_name", is("gogo")) - ) - ); - } - - @Test - public void testNotSupportedQuery() throws IOException { - exceptionRule.expect(ResponseException.class); - exceptionRule.expectMessage("Unsupported subquery"); - String query = String.format(Locale.ROOT, - "SELECT dog_name " + - "FROM %s A " + - "WHERE holdersName NOT IN (SELECT firstname FROM %s B WHERE age <> 36) " + - "AND dog_name <> 'babala'", - TEST_INDEX_DOGSUBQUERY, TEST_INDEX_ACCOUNT); - executeQuery(query); - } - - // todo Pending on DISTINCT support in JOIN - @Ignore - @Test - public void testINWithDuplicate() throws IOException { - String query = String.format(Locale.ROOT, - "SELECT dog_name " + - "FROM %s A " + - "WHERE holdersName IN (SELECT firstname FROM %s B)", - TEST_INDEX_DOGSUBQUERY, TEST_INDEX_ACCOUNT); - - JSONObject response = executeQuery(query); - assertThat( - response, - hitAll( - kvString("/_source/A.dog_name", is("snoopy")), - kvString("/_source/A.dog_name", is("babala")) - ) - ); - } - - @Test - public void nonCorrelatedExists() throws IOException { - String query = String.format(Locale.ROOT, - "SELECT e.name " + - "FROM %s as e " + - "WHERE EXISTS (SELECT * FROM e.projects as p)", - TEST_INDEX_EMPLOYEE_NESTED); - - JSONObject response = executeQuery(query); - assertThat( - response, - hitAll( - kvString("/_source/name", is("Bob Smith")), - kvString("/_source/name", is("Jane Smith")) - ) - ); - } - - @Test - public void nonCorrelatedExistsWhere() throws IOException { - String query = String.format(Locale.ROOT, - "SELECT e.name " + - "FROM %s as e " + - "WHERE EXISTS (SELECT * FROM e.projects as p WHERE p.name LIKE 'aurora')", - TEST_INDEX_EMPLOYEE_NESTED); - - JSONObject response = executeQuery(query); - assertThat( - response, - hitAll( - kvString("/_source/name", is("Bob Smith")) - ) - ); - } - - @Test - public void nonCorrelatedExistsParentWhere() throws IOException { - String query = String.format(Locale.ROOT, - "SELECT e.name " + - "FROM %s as e " + - "WHERE EXISTS (SELECT * FROM e.projects as p WHERE p.name LIKE 'security') " + - "AND e.name LIKE 'jane'", - TEST_INDEX_EMPLOYEE_NESTED); - - JSONObject response = executeQuery(query); - assertThat( - response, - hitAll( - kvString("/_source/name", is("Jane Smith")) - ) - ); - } - - @Test - public void nonCorrelatedNotExists() throws IOException { - String query = String.format(Locale.ROOT, - "SELECT e.name " + - "FROM %s as e " + - "WHERE NOT EXISTS (SELECT * FROM e.projects as p)", - TEST_INDEX_EMPLOYEE_NESTED); - - JSONObject response = executeQuery(query); - assertThat( - response, - hitAll( - kvString("/_source/name", is("Susan Smith")), - kvString("/_source/name", is("John Doe")) - ) - ); - } - - @Test - public void nonCorrelatedNotExistsWhere() throws IOException { - String query = String.format(Locale.ROOT, - "SELECT e.name " + - "FROM %s as e " + - "WHERE NOT EXISTS (SELECT * FROM e.projects as p WHERE p.name LIKE 'aurora')", - TEST_INDEX_EMPLOYEE_NESTED); - - JSONObject response = executeQuery(query); - assertThat( - response, - hitAll( - kvString("/_source/name", is("Susan Smith")), - kvString("/_source/name", is("Jane Smith")), - kvString("/_source/name", is("John Doe")) - ) - ); - } - - @Test - public void nonCorrelatedNotExistsParentWhere() throws IOException { - String query = String.format(Locale.ROOT, - "SELECT e.name " + - "FROM %s as e " + - "WHERE NOT EXISTS (SELECT * FROM e.projects as p WHERE p.name LIKE 'security') " + - "AND e.name LIKE 'smith'", - TEST_INDEX_EMPLOYEE_NESTED); - - JSONObject response = executeQuery(query); - assertThat( - response, - hitAll( - kvString("/_source/name", is("Susan Smith")) - ) - ); - } - - @Test - public void selectFromSubqueryWithCountShouldPass() throws IOException { - JSONObject result = executeQuery( - StringUtils.format("SELECT t.TEMP as count " + - "FROM (SELECT COUNT(*) as TEMP FROM %s) t", TEST_INDEX_ACCOUNT)); - - assertThat(result.query("/aggregations/count/value"), equalTo(1000)); - } - - @Test - public void selectFromSubqueryWithWhereAndCountShouldPass() throws IOException { - JSONObject result = executeQuery( - StringUtils.format("SELECT t.TEMP as count " + - "FROM (SELECT COUNT(*) as TEMP FROM %s WHERE age > 30) t", TEST_INDEX_ACCOUNT)); - - assertThat(result.query("/aggregations/count/value"), equalTo(502)); - } - - @Test - public void selectFromSubqueryWithCountAndGroupByShouldPass() throws Exception { - JSONObject result = executeQuery( - StringUtils.format("SELECT t.TEMP as count " + - "FROM (SELECT COUNT(*) as TEMP FROM %s GROUP BY gender) t", TEST_INDEX_ACCOUNT)); - - assertThat(getTotalHits(result), equalTo(1000)); - JSONObject gender = (JSONObject) result.query("/aggregations/gender"); - assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); - - boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); - int maleBucketId = isMaleFirst ? 0 : 1; - int femaleBucketId = isMaleFirst ? 1 : 0; - - String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); - String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); - - assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); - assertThat(gender.query(maleBucketPrefix + "/count/value"), equalTo(507)); - assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); - assertThat(gender.query(femaleBucketPrefix + "/count/value"), equalTo(493)); - } - - @Test - public void selectFromSubqueryWithCountAndGroupByAndOrderByShouldPass() throws IOException { - JSONObject result = executeQuery( - StringUtils.format( - "SELECT t.TEMP as count " + - "FROM (SELECT COUNT(*) as TEMP FROM %s GROUP BY age ORDER BY TEMP) t", - TEST_INDEX_ACCOUNT)); - JSONArray buckets = (JSONArray) result.query("/aggregations/age/buckets"); - List countList = new ArrayList<>(); - for (int i = 0; i < buckets.length(); ++i) { - countList.add((int) buckets.query(String.format(Locale.ROOT, "/%d/count/value", i))); - } - - assertTrue(Ordering.natural().isOrdered(countList)); - } - - @Test - public void selectFromSubqueryWithCountAndGroupByAndHavingShouldPass() throws Exception { - JSONObject result = executeQuery( - StringUtils.format("SELECT t.T1 as g, t.T2 as c " + - "FROM (SELECT gender as T1, COUNT(*) as T2 " + - " FROM %s " + - " GROUP BY gender " + - " HAVING T2 > 500) t", TEST_INDEX_ACCOUNT)); - assertThat(result.query("/aggregations/g/buckets/0/c/value"), equalTo(507)); - } - - @Test - public void selectFromSubqueryCountAndSum() throws IOException { - JSONObject result = executeQuery( - StringUtils.format( - "SELECT t.TEMP1 as count, t.TEMP2 as balance " + - "FROM (SELECT COUNT(*) as TEMP1, SUM(balance) as TEMP2 " + - " FROM %s) t", - TEST_INDEX_ACCOUNT)); - - assertThat(result.query("/aggregations/count/value"), equalTo(1000)); - assertThat(result.query("/aggregations/balance/value"), equalTo(25714837.0)); - } - - @Test - public void selectFromSubqueryWithoutAliasShouldPass() throws IOException { - JSONObject response = executeJdbcRequest( - StringUtils.format( - "SELECT a.firstname AS my_first, a.lastname AS my_last, a.age AS my_age " + - "FROM (SELECT firstname, lastname, age " + - "FROM %s " + - "WHERE age = 40 and account_number = 291) AS a", - TEST_INDEX_ACCOUNT)); - - verifySchema(response, - schema("firstname", "my_first", "text"), - schema("lastname", "my_last", "text"), - schema("age", "my_age", "long")); - verifyDataRows(response, - rows("Lynn", "Pollard", 40)); - } - - private JSONObject executeJdbcRequest(String query) { - return new JSONObject(executeQuery(query, "jdbc")); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TermQueryExplainIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TermQueryExplainIT.java deleted file mode 100644 index 5f20b6fee2..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TermQueryExplainIT.java +++ /dev/null @@ -1,419 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import org.elasticsearch.client.ResponseException; -import org.elasticsearch.rest.RestStatus; -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.IOException; -import java.util.Locale; - -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.not; - - - -public class TermQueryExplainIT extends SQLIntegTestCase { - - @Override - protected void init() throws Exception { - loadIndex(Index.ACCOUNT); - loadIndex(Index.ONLINE); - loadIndex(Index.BANK); - loadIndex(Index.BANK_TWO); - loadIndex(Index.DOG); - loadIndex(Index.DOGS2); - loadIndex(Index.DOGS3); - loadIndex(Index.EMPLOYEE_NESTED); - } - - @Test - public void testNonExistingIndex() throws IOException { - try { - explainQuery("SELECT firstname, lastname " + - "FROM elasticsearch_sql_test_fake_index " + - "WHERE firstname = 'Leo'"); - Assert.fail("Expected ResponseException, but none was thrown"); - - } catch (ResponseException e) { - assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.BAD_REQUEST.getStatus())); - final String entity = TestUtils.getResponseBody(e.getResponse()); - assertThat(entity, containsString("no such index")); - assertThat(entity, containsString("\"type\": \"IndexNotFoundException\"")); - } - } - - @Test - public void testNonResolvingIndexPattern() throws IOException { - try { - explainQuery("SELECT * " + - "FROM elasticsearch_sql_test_blah_blah* " + - "WHERE firstname = 'Leo'"); - Assert.fail("Expected ResponseException, but none was thrown"); - - } catch (ResponseException e) { - assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.BAD_REQUEST.getStatus())); - final String entity = TestUtils.getResponseBody(e.getResponse()); - assertThat(entity, containsString("Field [firstname] cannot be found or used here.")); - assertThat(entity, containsString("\"type\": \"SemanticAnalysisException\"")); - } - } - - @Test - public void testNonResolvingIndexPatternWithExistingIndex() throws IOException { - String result = explainQuery( - "SELECT * " + - "FROM elasticsearch_sql_test_blah_blah*, elasticsearch-sql_test_index_bank " + - "WHERE state = 'DC'"); - assertThat(result, containsString("\"term\":{\"state.keyword\"")); - } - - @Test - public void testNonResolvingIndexPatternWithNonExistingIndex() throws IOException { - try { - explainQuery( - "SELECT firstname, lastname " + - "FROM elasticsearch_sql_test_blah_blah*, another_fake_index " + - "WHERE firstname = 'Leo'"); - Assert.fail("Expected ResponseException, but none was thrown"); - } catch (ResponseException e) { - assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.BAD_REQUEST.getStatus())); - final String entity = TestUtils.getResponseBody(e.getResponse()); - assertThat(entity, containsString("no such index")); - assertThat(entity, containsString("\"type\": \"IndexNotFoundException\"")); - } - } - - @Test - public void testNonCompatibleMappings() throws IOException { - try { - explainQuery("SELECT * FROM elasticsearch-sql_test_index_dog, elasticsearch-sql_test_index_dog2"); - Assert.fail("Expected ResponseException, but none was thrown"); - } catch (ResponseException e) { - assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.BAD_REQUEST.getStatus())); - final String entity = TestUtils.getResponseBody(e.getResponse()); - assertThat(entity, containsString("Field [holdersName] have conflict type")); - assertThat(entity, containsString("\"type\": \"SemanticAnalysisException\"")); - } - } - - /** - * The dog_name field has same type in dog and dog2 index. - * But, the holdersName field has different type. - */ - @Test - public void testNonCompatibleMappingsButTheFieldIsNotUsed() throws IOException { - String result = explainQuery( - "SELECT dog_name " + - "FROM elasticsearch-sql_test_index_dog, elasticsearch-sql_test_index_dog2 WHERE dog_name = 'dog'"); - System.out.println(result); - assertThat(result, containsString("dog_name")); - assertThat(result, containsString("_source")); - } - - @Test - public void testEqualFieldMappings() throws IOException { - String result = explainQuery( - "SELECT color " + - "FROM elasticsearch-sql_test_index_dog2, elasticsearch-sql_test_index_dog3"); - assertThat(result, containsString("color")); - assertThat(result, containsString("_source")); - } - - @Test - public void testIdenticalMappings() throws IOException { - String result = explainQuery( - "SELECT firstname, birthdate, state " + - "FROM elasticsearch-sql_test_index_bank, elasticsearch-sql_test_index_bank_two " + - "WHERE state = 'WA' OR male = true" - ); - assertThat(result, containsString("term")); - assertThat(result, containsString("state.keyword")); - assertThat(result, containsString("_source")); - } - - @Test - public void testIdenticalMappingsWithTypes() throws IOException { - String result = explainQuery( - "SELECT firstname, birthdate, state " + - "FROM elasticsearch-sql_test_index_bank/account, elasticsearch-sql_test_index_bank_two/account_two " + - "WHERE state = 'WA' OR male = true" - ); - assertThat(result, containsString("term")); - assertThat(result, containsString("state.keyword")); - assertThat(result, containsString("_source")); - } - - - @Test - public void testIdenticalMappingsWithPartialType() throws IOException { - String result = explainQuery( - "SELECT firstname, birthdate, state " + - "FROM elasticsearch-sql_test_index_bank/account, elasticsearch-sql_test_index_bank_two " + - "WHERE state = 'WA' OR male = true" - ); - assertThat(result, containsString("term")); - assertThat(result, containsString("state.keyword")); - assertThat(result, containsString("_source")); - } - - @Test - public void testTextFieldOnly() throws IOException { - - String result = explainQuery( - "SELECT firstname, birthdate, state " + - "FROM elasticsearch-sql_test_index_bank " + - "WHERE firstname = 'Abbas'" - ); - assertThat(result, containsString("term")); - assertThat(result, not(containsString("firstname."))); - } - - @Test - public void testTextAndKeywordAppendsKeywordAlias() throws IOException { - String result = explainQuery( - "SELECT firstname, birthdate, state " + - "FROM elasticsearch-sql_test_index_bank " + - "WHERE state = 'WA' OR lastname = 'Chen'" - ); - assertThat(result, containsString("term")); - assertThat(result, containsString("state.keyword")); - assertThat(result, not(containsString("lastname."))); - } - - @Test - public void testBooleanFieldNoKeywordAlias() throws IOException { - - String result = explainQuery("SELECT * FROM elasticsearch-sql_test_index_bank WHERE male = false"); - assertThat(result, containsString("term")); - assertThat(result, not(containsString("male."))); - } - - @Test - public void testDateFieldNoKeywordAlias() throws IOException { - - String result = explainQuery("SELECT * FROM elasticsearch-sql_test_index_bank WHERE birthdate = '2018-08-19'"); - assertThat(result, containsString("term")); - assertThat(result, not(containsString("birthdate."))); - } - - @Test - public void testNumberNoKeywordAlias() throws IOException { - String result = explainQuery("SELECT * FROM elasticsearch-sql_test_index_bank WHERE age = 32"); - assertThat(result, containsString("term")); - assertThat(result, not(containsString("age."))); - } - - @Test - public void inTestInWhere() throws IOException { - String result = explainQuery( - "SELECT * " + - "FROM elasticsearch-sql_test_index_bank " + - "WHERE state IN ('WA' , 'PA' , 'TN')" - ); - assertThat(result, containsString("term")); - assertThat(result, containsString("state.keyword")); - } - - @Test - @Ignore // TODO: enable when subqueries are fixed - public void inTestInWhereSubquery() throws IOException { - String result = explainQuery( - "SELECT * " + - "FROM elasticsearch-sql_test_index_bank/account WHERE " + - "state IN (SELECT state FROM elasticsearch-sql_test_index_bank WHERE city = 'Nicholson')" - ); - assertThat(result, containsString("term")); - assertThat(result, containsString("state.keyword")); - } - - @Test - public void testKeywordAliasGroupBy() throws IOException { - String result = explainQuery( - "SELECT firstname, state " + - "FROM elasticsearch-sql_test_index_bank/account " + - "GROUP BY firstname, state"); - assertThat(result, containsString("term")); - assertThat(result, containsString("state.keyword")); - } - - @Test - public void testKeywordAliasGroupByUsingTableAlias() throws IOException { - String result = explainQuery( - "SELECT a.firstname, a.state " + - "FROM elasticsearch-sql_test_index_bank/account a " + - "GROUP BY a.firstname, a.state"); - assertThat(result, containsString("term")); - assertThat(result, containsString("state.keyword")); - } - - @Test - public void testKeywordAliasOrderBy() throws IOException { - String result = explainQuery( - "SELECT * " + - "FROM elasticsearch-sql_test_index_bank " + - "ORDER BY state, lastname " - ); - assertThat(result, containsString("\"state.keyword\":{\"order\":\"asc\"")); - assertThat(result, containsString("\"lastname\":{\"order\":\"asc\"}")); - } - - @Test - public void testKeywordAliasOrderByUsingTableAlias() throws IOException { - String result = explainQuery( - "SELECT * " + - "FROM elasticsearch-sql_test_index_bank b " + - "ORDER BY b.state, b.lastname " - ); - assertThat(result, containsString("\"state.keyword\":{\"order\":\"asc\"")); - assertThat(result, containsString("\"lastname\":{\"order\":\"asc\"}")); - } - - @Test - @Ignore // TODO: verify the returned query is correct and fix the expected output - public void testJoinWhere() throws IOException { - String expectedOutput = TestUtils.fileToString("src/test/resources/expectedOutput/term_join_where", true); - String result = explainQuery( - "SELECT a.firstname, a.lastname , b.city " + - "FROM elasticsearch-sql_test_index_account a " + - "JOIN elasticsearch-sql_test_index_account b " + - "ON a.city = b.city " + - "WHERE a.city IN ('Nicholson', 'Yardville')" - ); - - assertThat(result.replaceAll("\\s+",""), equalTo(expectedOutput.replaceAll("\\s+",""))); - } - - @Test - public void testJoinAliasMissing() throws IOException { - try { - explainQuery( - "SELECT a.firstname, a.lastname , b.city " + - "FROM elasticsearch-sql_test_index_account a " + - "JOIN elasticsearch-sql_test_index_account b " + - "ON a.city = b.city " + - "WHERE city IN ('Nicholson', 'Yardville')" - ); - Assert.fail("Expected ResponseException, but none was thrown"); - } catch (ResponseException e) { - assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.BAD_REQUEST.getStatus())); - final String entity = TestUtils.getResponseBody(e.getResponse()); - assertThat(entity, containsString("Field name [city] is ambiguous")); - assertThat(entity, containsString("\"type\": \"VerificationException\"")); - } - - } - - @Test - public void testNestedSingleConditionAllFields() throws IOException { - String result = explainQuery( - "SELECT * " + - "FROM elasticsearch-sql_test_index_employee_nested e, e.projects p " + - "WHERE p.name = 'something' " - ); - assertThat(result, containsString("\"term\":{\"projects.name.keyword\":{\"value\":\"something\"")); - assertThat(result, containsString("\"path\":\"projects\"")); - } - - @Test - public void testNestedMultipleCondition() throws IOException { - String result = explainQuery( - "SELECT e.id, p.name " + - "FROM elasticsearch-sql_test_index_employee_nested e, e.projects p " + - "WHERE p.name = 'something' and p.started_year = 1990 " - ); - assertThat(result, containsString("\"term\":{\"projects.name.keyword\":{\"value\":\"something\"")); - assertThat(result, containsString("\"term\":{\"projects.started_year\":{\"value\":1990")); - assertThat(result, containsString("\"path\":\"projects\"")); - } - - @Test - public void testConditionsOnDifferentNestedDocs() throws IOException { - String result = explainQuery( - "SELECT p.name, c.likes " + - "FROM elasticsearch-sql_test_index_employee_nested e, e.projects p, e.comments c " + - "WHERE p.name = 'something' or c.likes = 56 " - ); - assertThat(result, containsString("\"term\":{\"projects.name.keyword\":{\"value\":\"something\"")); - assertThat(result, containsString("\"term\":{\"comments.likes\":{\"value\":56")); - assertThat(result, containsString("\"path\":\"projects\"")); - assertThat(result, containsString("\"path\":\"comments\"")); - } - - @Test - public void testNestedSingleConditionSpecificFields() throws IOException { - String result = explainQuery( - "SELECT e.id, p.name " + - "FROM elasticsearch-sql_test_index_employee_nested e, e.projects p " + - "WHERE p.name = 'hello' or p.name = 'world' " - ); - assertThat(result, containsString("\"term\":{\"projects.name.keyword\":{\"value\":\"hello\"")); - assertThat(result, containsString("\"term\":{\"projects.name.keyword\":{\"value\":\"world\"")); - assertThat(result, containsString("\"path\":\"projects\"")); - } - - @Test - public void testNestedSingleGroupBy() throws IOException { - String result = explainQuery( - "SELECT e.id, p.name " + - "FROM elasticsearch-sql_test_index_employee_nested e, e.projects p " + - "GROUP BY p.name "); - assertThat(result, containsString("\"terms\":{\"field\":\"projects.name.keyword\"")); - assertThat(result, containsString("\"nested\":{\"path\":\"projects\"")); - } - - @Test - public void testNestedSingleOrderBy() throws IOException { - String result = explainQuery( - "SELECT e.id, p.name " + - "FROM elasticsearch-sql_test_index_employee_nested e, e.projects p " + - "ORDER BY p.name " - ); - assertThat(result, containsString("\"sort\":[{\"projects.name.keyword\"")); - assertThat(result, containsString("\"nested\":{\"path\":\"projects\"")); - } - - @Test - public void testNestedIsNotNullExplain() throws IOException { - String explain = explainQuery( - "SELECT e.name " + - "FROM elasticsearch-sql_test_index_employee_nested as e, e.projects as p " + - "WHERE p IS NOT NULL" - ); - - assertThat(explain, containsString("\"exists\":{\"field\":\"projects\"")); - assertThat(explain, containsString("\"path\":\"projects\"")); - } - - @Test - @Ignore // TODO: enable when subqueries are fixed - public void testMultiQuery() throws IOException { - String expectedOutput = TestUtils.fileToString("src/test/resources/expectedOutput/term_union_where", true); - String result = explainQuery( - "SELECT firstname " + - "FROM elasticsearch-sql_test_index_account/account " + - "WHERE firstname = 'Amber' " + - "UNION ALL " + - "SELECT dog_name as firstname " + - "FROM elasticsearch-sql_test_index_dog/dog " + - "WHERE holdersName = 'Hattie' OR dog_name = 'rex'"); - assertThat(result.replaceAll("\\s+",""), equalTo(expectedOutput.replaceAll("\\s+",""))); - } -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TypeInformationIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TypeInformationIT.java deleted file mode 100644 index 80adad1ea5..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TypeInformationIT.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.esintgtest; - -import org.json.JSONObject; -import org.junit.Test; - -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.schema; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifySchema; - -public class TypeInformationIT extends SQLIntegTestCase { - - @Override - protected void init() throws Exception { - loadIndex(Index.ACCOUNT); - loadIndex(Index.ONLINE); - } - - /* - numberOperators - */ - @Test - public void testAbsWithIntFieldReturnsInt() { - JSONObject response = executeJdbcRequest("SELECT ABS(age) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + - " ORDER BY age LIMIT 5"); - - verifySchema(response, schema("ABS(age)", null, "long")); - } - - @Test - public void testCeilWithLongFieldReturnsLong() { - JSONObject response = executeJdbcRequest("SELECT CEIL(balance) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + - " ORDER BY balance LIMIT 5"); - - verifySchema(response, schema("CEIL(balance)", null, "long")); - } - - /* - mathConstants - */ - @Test - public void testPiReturnsDouble() { - JSONObject response = executeJdbcRequest("SELECT PI() FROM " + TestsConstants.TEST_INDEX_ACCOUNT - + " LIMIT 1"); - - verifySchema(response, schema("PI()", null, "double")); - } - - /* - stringOperators - */ - @Test - public void testUpperWithStringFieldReturnsString() { - JSONObject response = executeJdbcRequest("SELECT UPPER(firstname) AS firstname_alias FROM " + - TestsConstants.TEST_INDEX_ACCOUNT + " ORDER BY firstname_alias LIMIT 2"); - - verifySchema(response, schema("firstname_alias", null, "text")); - } - - @Test - public void testLowerWithTextFieldReturnsText() { - JSONObject response = executeJdbcRequest("SELECT LOWER(firstname) FROM " + - TestsConstants.TEST_INDEX_ACCOUNT + " ORDER BY firstname LIMIT 2"); - - verifySchema(response, schema("LOWER(firstname)", null, "text")); - } - - /* - stringFunctions - */ - @Test - public void testLengthWithTextFieldReturnsInt() { - JSONObject response = executeJdbcRequest("SELECT length(firstname) FROM " + - TestsConstants.TEST_INDEX_ACCOUNT + " ORDER BY firstname LIMIT 2"); - - verifySchema(response, schema("length(firstname)", null, "integer")); - } - - @Test - public void testLengthWithGroupByExpr() { - JSONObject response = executeJdbcRequest("SELECT Length(firstname) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + - " GROUP BY LENGTH(firstname) LIMIT 5"); - - verifySchema(response, schema("Length(firstname)", null, "integer")); - } - - /* - trigFunctions - */ - @Test - public void testSinWithLongFieldReturnsDouble() { - JSONObject response = executeJdbcRequest("SELECT sin(balance) FROM " + - TestsConstants.TEST_INDEX_ACCOUNT + " ORDER BY firstname LIMIT 2"); - - verifySchema(response, schema("sin(balance)", null, "double")); - } - - @Test - public void testRadiansWithLongFieldReturnsDouble() { - JSONObject response = executeJdbcRequest("SELECT radians(balance) FROM " + - TestsConstants.TEST_INDEX_ACCOUNT + " ORDER BY firstname LIMIT 2"); - - verifySchema(response, schema("radians(balance)", null, "double")); - } - - /* - binaryOperators - */ - @Test - public void testAddWithIntReturnsInt() { - JSONObject response = executeJdbcRequest("SELECT (balance + 5) AS balance_add_five FROM " + - TestsConstants.TEST_INDEX_ACCOUNT + " ORDER BY firstname LIMIT 2"); - - verifySchema(response, schema("balance_add_five", null, "integer")); - } - - @Test - public void testSubtractLongWithLongReturnsLong() { - JSONObject response = executeJdbcRequest("SELECT (balance - balance) FROM " + - TestsConstants.TEST_INDEX_ACCOUNT + " ORDER BY firstname LIMIT 2"); - - verifySchema(response, schema("subtract(balance, balance)", null, "long")); - } - - /* - dateFunctions - */ - @Test - public void testDayOfWeekWithKeywordReturnsText() { - JSONObject response = executeJdbcRequest("SELECT DAY_OF_WEEK(insert_time) FROM " - + TestsConstants.TEST_INDEX_ONLINE + " LIMIT 2"); - - verifySchema(response, - schema("DAY_OF_WEEK(insert_time)", null, "integer")); - } - - @Test - public void testYearWithKeywordReturnsText() { - JSONObject response = executeJdbcRequest("SELECT YEAR(insert_time) FROM " - + TestsConstants.TEST_INDEX_ONLINE + " LIMIT 2"); - - verifySchema(response, schema("YEAR(insert_time)", null, "integer")); - } - - private JSONObject executeJdbcRequest(String query) { - return new JSONObject(executeQuery(query, "jdbc")); - } - -} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/executor/format/BindingTupleResultSetTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/executor/format/BindingTupleResultSetTest.java deleted file mode 100644 index cf51d5cd44..0000000000 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/executor/format/BindingTupleResultSetTest.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.opendistroforelasticsearch.sql.unittest.executor.format; - -import com.amazon.opendistroforelasticsearch.sql.executor.format.BindingTupleResultSet; -import com.amazon.opendistroforelasticsearch.sql.executor.format.DataRows; -import com.amazon.opendistroforelasticsearch.sql.expression.domain.BindingTuple; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import org.hamcrest.Matcher; -import org.junit.Test; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.featureValueOf; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.hasEntry; - -public class BindingTupleResultSetTest { - - @Test - public void buildDataRowsFromBindingTupleShouldPass() { - assertThat(row(Arrays.asList(BindingTuple.from(ImmutableMap.of("age", 31, "gender", "m")), - BindingTuple.from(ImmutableMap.of("age", 31, "gender", "f")), - BindingTuple.from(ImmutableMap.of("age", 39, "gender", "m")), - BindingTuple.from(ImmutableMap.of("age", 39, "gender", "f")))), - containsInAnyOrder(rowContents(allOf(hasEntry("age", 31), hasEntry("gender", (Object) "m"))), - rowContents(allOf(hasEntry("age", 31), hasEntry("gender", (Object) "f"))), - rowContents(allOf(hasEntry("age", 39), hasEntry("gender", (Object) "m"))), - rowContents(allOf(hasEntry("age", 39), hasEntry("gender", (Object) "f"))))); - } - - private static Matcher rowContents(Matcher> matcher) { - return featureValueOf("DataRows.Row", matcher, DataRows.Row::getContents); - } - - private List row(List bindingTupleList) { - return ImmutableList.copyOf(BindingTupleResultSet.buildDataRows(bindingTupleList).iterator()); - } -}