From 88d4ce5243ae7afea7c42220756abeef002c4194 Mon Sep 17 00:00:00 2001 From: Peng Date: Mon, 13 May 2019 22:07:50 -0700 Subject: [PATCH 1/2] Support mvn build and integration-test for Elasticsearch 7.0.1 --- build.gradle | 2 +- pom.xml | 4 +- .../sql/executor/join/ElasticUtils.java | 5 +- .../sql/utils/SQLFunctions.java | 2 +- .../sql/intgtest/AggregationTest.java | 4 +- .../sql/intgtest/DateFormatTest.java | 22 +++-- .../sql/intgtest/DeleteTest.java | 4 +- .../sql/intgtest/MainTestSuite.java | 2 +- .../sql/intgtest/PreparedStatementTest.java | 6 +- .../sql/intgtest/QueryTest.java | 94 +++++++++---------- .../sql/intgtest/SQLFunctionsTest.java | 2 +- .../sql/intgtest/TestsConstants.java | 2 +- .../sql/unittest/DateFunctionsTest.java | 14 +-- src/test/resources/join_objects.json | 10 +- 14 files changed, 92 insertions(+), 81 deletions(-) diff --git a/build.gradle b/build.gradle index f997e355e3..948f3a2fad 100644 --- a/build.gradle +++ b/build.gradle @@ -194,7 +194,7 @@ afterEvaluate { url 'https://opendistro.github.io/elasticsearch/downloads' summary ''' SQL plugin for OpenDistro for Elasticsearch. - Reference documentation can be found at https://opendistro.github.io/elasticsearch/docs. + Reference documentation can be found at https://opendistro.github.io/for-elasticsearch-docs/. '''.stripIndent().replace('\n', ' ').trim() } diff --git a/pom.xml b/pom.xml index 9a3c3b0fd9..c42357c591 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.opendistroforelasticsearch opendistro-sql - 6.7.1.0 + 7.0.1.0 jar Open Distro for Elasticsearch SQL opendistro-sql @@ -27,7 +27,7 @@ UTF-8 **/MainTestSuite.class opendistro-sql - 6.7.1 + 7.0.1 com.amazon.opendistroforelasticsearch.sql.plugin.SqlPlug diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/ElasticUtils.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/ElasticUtils.java index 26b9a48c65..b27e988b4a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/ElasticUtils.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/ElasticUtils.java @@ -122,7 +122,10 @@ public static XContentBuilder hitsAsStringResultZeroCopy(List results /** Code copy from SearchHits */ private static void toXContent(XContentBuilder builder, Params params, List hits, ElasticJoinExecutor executor) throws IOException { builder.startObject(SearchHits.Fields.HITS); - builder.field(SearchHits.Fields.TOTAL, hits.size()); + builder.field(SearchHits.Fields.TOTAL, ImmutableMap.of( + "value", hits.size(), + "relation", Relation.EQUAL_TO + )); builder.field(SearchHits.Fields.MAX_SCORE, 1.0f); builder.field(SearchHits.Fields.HITS); builder.startArray(); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SQLFunctions.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SQLFunctions.java index 9821d03e60..f14dd420e8 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SQLFunctions.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SQLFunctions.java @@ -334,7 +334,7 @@ private static Tuple date_format(SQLExpr field, String pattern, private static Tuple dateFunctionTemplate(String methodName, SQLExpr field) { String name = randomize(methodName); - return new Tuple<>(name, def(name, doc(field) + ".date." + methodName)); + return new Tuple<>(name, def(name, doc(field) + ".value." + methodName)); } public static Tuple add(SQLExpr a, SQLExpr b) { diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/AggregationTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/AggregationTest.java index 92dddbd822..9242189784 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/AggregationTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/AggregationTest.java @@ -554,7 +554,7 @@ public void testSubAggregations() throws Exception { } } - Assert.assertEquals(response.getHits().getTotalHits(), 1000); + Assert.assertEquals(response.getHits().getTotalHits().value, 1000); Assert.assertEquals(response.getHits().getHits().length, 10); } @@ -581,7 +581,7 @@ public void testSimpleSubAggregations() throws Exception { } } - Assert.assertEquals(response.getHits().getTotalHits(), 1000); + Assert.assertEquals(response.getHits().getTotalHits().value, 1000); Assert.assertEquals(response.getHits().getHits().length, 10); } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/DateFormatTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/DateFormatTest.java index 79285c69fe..f62c858f15 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/DateFormatTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/DateFormatTest.java @@ -50,7 +50,7 @@ public class DateFormatTest { @Test public void equalTo() { assertThat( - query(SELECT_FROM + "WHERE date_format(insert_time, 'YYYY-MM-dd', 'UTC') = '2014-08-17'"), + query(SELECT_FROM + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') = '2014-08-17'"), containsInAnyOrder("2014-08-17") ); } @@ -58,7 +58,7 @@ public void equalTo() { @Test public void lessThan() { assertThat( - query(SELECT_FROM + "WHERE date_format(insert_time, 'YYYY-MM-dd', 'UTC') < '2014-08-18'"), + query(SELECT_FROM + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') < '2014-08-18'"), containsInAnyOrder("2014-08-17") ); } @@ -66,7 +66,9 @@ public void lessThan() { @Test public void lessThanOrEqualTo() { assertThat( - query(SELECT_FROM + "WHERE date_format(insert_time, 'YYYY-MM-dd', 'UTC') <= '2014-08-18'"), + query(SELECT_FROM + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') <= '2014-08-18' " + + "ORDER BY insert_time " + + "LIMIT 1000"), containsInAnyOrder("2014-08-17", "2014-08-18") ); } @@ -74,7 +76,7 @@ public void lessThanOrEqualTo() { @Test public void greaterThan() { assertThat( - query(SELECT_FROM + "WHERE date_format(insert_time, 'YYYY-MM-dd', 'UTC') > '2014-08-23'"), + query(SELECT_FROM + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') > '2014-08-23'"), containsInAnyOrder("2014-08-24") ); } @@ -86,7 +88,7 @@ public void greaterThan() { @Test public void greaterThanOrEqualTo() { assertThat( - query(SELECT_FROM + "WHERE date_format(insert_time, 'YYYY-MM-dd', 'UTC') >= '2014-08-23' LIMIT 1000"), + query(SELECT_FROM + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') >= '2014-08-23' LIMIT 2000"), containsInAnyOrder("2014-08-23", "2014-08-24") ); } @@ -95,8 +97,8 @@ public void greaterThanOrEqualTo() { public void and() { assertThat( query(SELECT_FROM + - "WHERE date_format(insert_time, 'YYYY-MM-dd', 'UTC') >= '2014-08-21' " + - "AND date_format(insert_time, 'YYYY-MM-dd', 'UTC') <= '2014-08-23' LIMIT 1000"), + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') >= '2014-08-21' " + + "AND date_format(insert_time, 'yyyy-MM-dd', 'UTC') <= '2014-08-23' LIMIT 3000"), containsInAnyOrder("2014-08-21", "2014-08-22", "2014-08-23") ); } @@ -105,8 +107,10 @@ public void and() { public void or() { assertThat( query(SELECT_FROM + - "WHERE date_format(insert_time, 'YYYY-MM-dd', 'UTC') < '2014-08-18' " + - "OR date_format(insert_time, 'YYYY-MM-dd', 'UTC') > '2014-08-23'"), + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') < '2014-08-18' " + + "OR date_format(insert_time, 'yyyy-MM-dd', 'UTC') > '2014-08-23'" + + "ORDER BY insert_time " + + "LIMIT 3000"), containsInAnyOrder("2014-08-17", "2014-08-24") ); } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/DeleteTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/DeleteTest.java index 05c1c8d120..6fe6d4804e 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/DeleteTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/DeleteTest.java @@ -53,7 +53,7 @@ public void deleteAllTest() throws SqlParseException, SQLFeatureNotSupportedExce SearchRequestBuilder request = MainTestSuite.getClient().prepareSearch(TEST_INDEX_ACCOUNT_TEMP); request.setTypes("temp_account"); SearchResponse response = request.setQuery(QueryBuilders.matchAllQuery()).get(); - assertThat(response.getHits().getTotalHits(), equalTo(0L)); + assertThat(response.getHits().getTotalHits().value, equalTo(0L)); } @@ -64,7 +64,7 @@ public void deleteWithConditionTest() throws SqlParseException, SQLFeatureNotSup SearchRequestBuilder request = MainTestSuite.getClient().prepareSearch(TEST_INDEX_PHRASE); request.setTypes("phrase"); SearchResponse response = request.setQuery(QueryBuilders.matchAllQuery()).get(); - assertThat(response.getHits().getTotalHits(), equalTo(5L)); + assertThat(response.getHits().getTotalHits().value, equalTo(5L)); } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/MainTestSuite.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/MainTestSuite.java index 5fe8bfc18a..2386ebab58 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/MainTestSuite.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/MainTestSuite.java @@ -514,7 +514,7 @@ public static void prepareOdbcIndex(){ "\t\t\"properties\":{\n" + "\t\t\t\"odbc_time\":{\n" + "\t\t\t\t\"type\":\"date\",\n" + - "\t\t\t\t\"format\": \"{'ts' ''yyyy-MM-dd HH:mm:ss.SSS''}\"\n" + + "\t\t\t\t\"format\": \"'{ts' ''yyyy-MM-dd HH:mm:ss.SSS'''}'\"\n" + "\t\t\t},\n" + "\t\t\t\"docCount\":{\n" + "\t\t\t\t\"type\":\"text\"\n" + diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/PreparedStatementTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/PreparedStatementTest.java index b12a4488a4..510746d8ad 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/PreparedStatementTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/PreparedStatementTest.java @@ -23,6 +23,7 @@ import com.amazon.opendistroforelasticsearch.sql.request.SqlRequest; import com.amazon.opendistroforelasticsearch.sql.request.SqlRequestFactory; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.http.HttpChannel; import org.elasticsearch.http.HttpRequest; @@ -34,6 +35,7 @@ import java.sql.SQLFeatureNotSupportedException; +import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.greaterThan; import static org.junit.Assert.assertTrue; @@ -78,7 +80,9 @@ public void search() { private SearchHits query(String request) { try { final HttpRequest httpRequest = mock(HttpRequest.class); - when (httpRequest.uri()).thenReturn(""); + when(httpRequest.uri()).thenReturn("uri"); + when(httpRequest.method()).thenReturn(POST); + when(httpRequest.content()).thenReturn(new BytesArray(request)); SqlRequest sqlRequest = SqlRequestFactory.getSqlRequest(RestRequest.request(NamedXContentRegistry.EMPTY, httpRequest, mock(HttpChannel.class))); diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/QueryTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/QueryTest.java index f94112dabf..ca4a77c313 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/QueryTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/QueryTest.java @@ -48,14 +48,14 @@ public class QueryTest { @Test public void searchTypeTest() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT * FROM %s/phrase LIMIT 1000", TEST_INDEX_PHRASE)); - Assert.assertEquals(6, response.getTotalHits()); + Assert.assertEquals(6, response.getTotalHits().value); } @Test public void multipleFromTest() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT * FROM %s/account, %s/account_two LIMIT 2000", TEST_INDEX_BANK, TEST_INDEX_BANK_TWO)); - Assert.assertEquals(14, response.getTotalHits()); + Assert.assertEquals(14, response.getTotalHits().value); } @Test @@ -112,7 +112,7 @@ public void equallityTest() throws SqlParseException, SQLFeatureNotSupportedExce SearchHit[] hits = response.getHits(); // assert the results is correct according to accounts.json data. - Assert.assertEquals(1, response.getTotalHits()); + Assert.assertEquals(1, response.getTotalHits().value); Assert.assertEquals("Nogal", hits[0].getSourceAsMap().get("city")); } @@ -125,7 +125,7 @@ public void equallityTest_phrase() throws SqlParseException, SQLFeatureNotSuppor SearchHit[] hits = response.getHits(); // assert the results is correct according to accounts.json data. - Assert.assertEquals(1, response.getTotalHits()); + Assert.assertEquals(1, response.getTotalHits().value); Assert.assertEquals("quick fox here", hits[0].getSourceAsMap().get("phrase")); } @@ -196,7 +196,7 @@ public void lessThanOrEqualTest() throws IOException, SqlParseException, SQLFeat public void orTest() throws IOException, SqlParseException, SQLFeatureNotSupportedException { SearchHits response = query(String.format("SELECT * FROM %s/account WHERE match_phrase(gender, 'F') OR match_phrase(gender, 'M') LIMIT 1000", TEST_INDEX_ACCOUNT)); // Assert all documents from accounts.json is returned. - Assert.assertEquals(1000, response.getTotalHits()); + Assert.assertEquals(1000, response.getTotalHits().value); } @@ -217,7 +217,7 @@ public void likeTest() throws IOException, SqlParseException, SQLFeatureNotSuppo SearchHit[] hits = response.getHits(); // assert the results is correct according to accounts.json data. - Assert.assertEquals(1, response.getTotalHits()); + Assert.assertEquals(1, response.getTotalHits().value); Assert.assertEquals("Amber", hits[0].getSourceAsMap().get("firstname")); } @@ -227,7 +227,7 @@ public void notLikeTest() throws IOException, SqlParseException, SQLFeatureNotSu SearchHit[] hits = response.getHits(); // assert we got hits - Assert.assertNotEquals(0, response.getTotalHits()); + Assert.assertNotEquals(0, response.getTotalHits().value); for (SearchHit hit : hits) { Assert.assertFalse(hit.getSourceAsMap().get("firstname").toString().toLowerCase().startsWith("amb")); } @@ -247,23 +247,23 @@ public void regexpQueryTest() throws SqlParseException, SQLFeatureNotSupportedEx @Test public void doubleNotTest() throws IOException, SqlParseException, SQLFeatureNotSupportedException { SearchHits response1 = query(String.format("SELECT * FROM %s/account WHERE not gender like 'm' and not gender like 'f'", TEST_INDEX_ACCOUNT)); - Assert.assertEquals(0, response1.getTotalHits()); + Assert.assertEquals(0, response1.getTotalHits().value); SearchHits response2 = query(String.format("SELECT * FROM %s/account WHERE not gender like 'm' and gender not like 'f'", TEST_INDEX_ACCOUNT)); - Assert.assertEquals(0, response2.getTotalHits()); + Assert.assertEquals(0, response2.getTotalHits().value); SearchHits response3 = query(String.format("SELECT * FROM %s/account WHERE gender not like 'm' and gender not like 'f'", TEST_INDEX_ACCOUNT)); - Assert.assertEquals(0, response3.getTotalHits()); + Assert.assertEquals(0, response3.getTotalHits().value); SearchHits response4 = query(String.format("SELECT * FROM %s/account WHERE gender like 'm' and not gender like 'f'", TEST_INDEX_ACCOUNT)); // assert there are results and they all have gender 'm' - Assert.assertNotEquals(0, response4.getTotalHits()); + Assert.assertNotEquals(0, response4.getTotalHits().value); for (SearchHit hit : response4.getHits()) { Assert.assertEquals("m", hit.getSourceAsMap().get("gender").toString().toLowerCase()); } SearchHits response5 = query(String.format("SELECT * FROM %s/account WHERE NOT (gender = 'm' OR gender = 'f')", TEST_INDEX_ACCOUNT)); - Assert.assertEquals(0, response5.getTotalHits()); + Assert.assertEquals(0, response5.getTotalHits().value); } @Test @@ -326,7 +326,7 @@ public void inTest() throws IOException, SqlParseException, SQLFeatureNotSupport public void inTestWithStrings() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT phrase FROM %s/phrase WHERE phrase IN ('quick', 'fox') LIMIT 1000", TEST_INDEX_PHRASE)); SearchHit[] hits = response.getHits(); - Assert.assertEquals(4, response.getTotalHits()); + Assert.assertEquals(4, response.getTotalHits().value); for(SearchHit hit : hits) { String phrase = (String) hit.getSourceAsMap().get("phrase"); assertThat(phrase, isOneOf("quick fox here", "fox brown", "quick fox" , "brown fox")); @@ -337,7 +337,7 @@ public void inTestWithStrings() throws IOException, SqlParseException, SQLFeatur public void inTermsTestWithIdentifiersTreatLikeStrings() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT name FROM %s/gotCharacters WHERE name.firstname = IN_TERMS(daenerys,eddard) LIMIT 1000", TEST_INDEX_GAME_OF_THRONES)); SearchHit[] hits = response.getHits(); - Assert.assertEquals(2, response.getTotalHits()); + Assert.assertEquals(2, response.getTotalHits().value); for(SearchHit hit : hits) { String firstname = ((Map) hit.getSourceAsMap().get("name")).get("firstname").toString(); assertThat(firstname, isOneOf("Daenerys", "Eddard")); @@ -347,7 +347,7 @@ public void inTermsTestWithIdentifiersTreatLikeStrings() throws IOException, Sql public void inTermsTestWithStrings() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT name FROM %s/gotCharacters WHERE name.firstname = IN_TERMS('daenerys','eddard') LIMIT 1000", TEST_INDEX_GAME_OF_THRONES)); SearchHit[] hits = response.getHits(); - Assert.assertEquals(2, response.getTotalHits()); + Assert.assertEquals(2, response.getTotalHits().value); for(SearchHit hit : hits) { String firstname = ((Map) hit.getSourceAsMap().get("name")).get("firstname").toString(); assertThat(firstname, isOneOf("Daenerys", "Eddard")); @@ -358,7 +358,7 @@ public void inTermsTestWithStrings() throws IOException, SqlParseException, SQLF public void inTermsTestWithNumbers() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT name FROM %s/gotCharacters WHERE name.ofHisName = IN_TERMS(4,2) LIMIT 1000", TEST_INDEX_GAME_OF_THRONES)); SearchHit[] hits = response.getHits(); - Assert.assertEquals(1, response.getTotalHits()); + Assert.assertEquals(1, response.getTotalHits().value); SearchHit hit = hits[0]; String firstname = ((Map) hit.getSourceAsMap().get("name")).get("firstname").toString(); Assert.assertEquals("Brandon",firstname); @@ -369,7 +369,7 @@ public void inTermsTestWithNumbers() throws IOException, SqlParseException, SQLF public void termQueryWithNumber() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT name FROM %s/gotCharacters WHERE name.ofHisName = term(4) LIMIT 1000", TEST_INDEX_GAME_OF_THRONES)); SearchHit[] hits = response.getHits(); - Assert.assertEquals(1, response.getTotalHits()); + Assert.assertEquals(1, response.getTotalHits().value); SearchHit hit = hits[0]; String firstname = ((Map) hit.getSourceAsMap().get("name")).get("firstname").toString(); Assert.assertEquals("Brandon",firstname); @@ -379,7 +379,7 @@ public void termQueryWithNumber() throws IOException, SqlParseException, SQLFeat public void termQueryWithStringIdentifier() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT name FROM %s/gotCharacters WHERE name.firstname = term(brandon) LIMIT 1000", TEST_INDEX_GAME_OF_THRONES)); SearchHit[] hits = response.getHits(); - Assert.assertEquals(1, response.getTotalHits()); + Assert.assertEquals(1, response.getTotalHits().value); SearchHit hit = hits[0]; String firstname = ((Map) hit.getSourceAsMap().get("name")).get("firstname").toString(); Assert.assertEquals("Brandon",firstname); @@ -389,7 +389,7 @@ public void termQueryWithStringIdentifier() throws IOException, SqlParseExceptio public void termQueryWithStringLiteral() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT name FROM %s/gotCharacters WHERE name.firstname = term('brandon') LIMIT 1000", TEST_INDEX_GAME_OF_THRONES)); SearchHit[] hits = response.getHits(); - Assert.assertEquals(1, response.getTotalHits()); + Assert.assertEquals(1, response.getTotalHits().value); SearchHit hit = hits[0]; String firstname = ((Map) hit.getSourceAsMap().get("name")).get("firstname").toString(); Assert.assertEquals("Brandon",firstname); @@ -479,7 +479,7 @@ public void missFilterSearch() throws IOException, SqlParseException, SQLFeature SearchHit[] hits = response.getHits(); // should be 2 according to the data. - Assert.assertEquals(response.getTotalHits(), 4); + Assert.assertEquals(response.getTotalHits().value, 4); for(SearchHit hit : hits) { assertThat(hit.getSourceAsMap(), not(hasKey("insert_time2"))); } @@ -491,7 +491,7 @@ public void notMissFilterSearch() throws IOException, SqlParseException, SQLFeat SearchHit[] hits = response.getHits(); // should be 2 according to the data. - Assert.assertEquals(response.getTotalHits(), 2); + Assert.assertEquals(response.getTotalHits().value, 2); for(SearchHit hit : hits) { assertThat(hit.getSourceAsMap(), hasKey("insert_time2")); } @@ -585,25 +585,25 @@ public void orderByAscFieldWithSpaceTest() throws IOException, SqlParseException @Test public void testMultipartWhere() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT * FROM %s/account WHERE (firstname LIKE 'opal' OR firstname like 'rodriquez') AND (state like 'oh' OR state like 'hi')", TEST_INDEX_ACCOUNT)); - Assert.assertEquals(2, response.getTotalHits()); + Assert.assertEquals(2, response.getTotalHits().value); } @Test public void testMultipartWhere2() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT * FROM %s/account where ((account_number > 200 and account_number < 300) or gender like 'm') and (state like 'hi' or address like 'avenue')", TEST_INDEX_ACCOUNT)); - Assert.assertEquals(127, response.getTotalHits()); + Assert.assertEquals(127, response.getTotalHits().value); } @Test public void testMultipartWhere3() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT * FROM %s/account where ((account_number > 25 and account_number < 75) and age >35 ) and (state like 'md' or (address like 'avenue' or address like 'street'))", TEST_INDEX_ACCOUNT)); - Assert.assertEquals(7, response.getTotalHits()); + Assert.assertEquals(7, response.getTotalHits().value); } @Test public void filterPolygonTest() throws SQLFeatureNotSupportedException, SqlParseException, InterruptedException { SearchHits results = query(String.format("SELECT * FROM %s/location WHERE GEO_INTERSECTS(place,'POLYGON ((102 2, 103 2, 103 3, 102 3, 102 2))')", TEST_INDEX_LOCATION)); - org.junit.Assert.assertEquals(1,results.getTotalHits()); + org.junit.Assert.assertEquals(1,results.getTotalHits().value); SearchHit result = results.getAt(0); Assert.assertEquals("bigSquare",result.getSourceAsMap().get("description")); } @@ -611,14 +611,14 @@ public void filterPolygonTest() throws SQLFeatureNotSupportedException, SqlParse @Test public void boundingBox() throws SQLFeatureNotSupportedException, SqlParseException, InterruptedException { SearchHits results = query(String.format("SELECT * FROM %s/location WHERE GEO_BOUNDING_BOX(center,100.0,1.0,101,0.0)", TEST_INDEX_LOCATION)); - org.junit.Assert.assertEquals(1,results.getTotalHits()); + org.junit.Assert.assertEquals(1,results.getTotalHits().value); SearchHit result = results.getAt(0); Assert.assertEquals("square",result.getSourceAsMap().get("description")); } @Test public void geoDistance() throws SQLFeatureNotSupportedException, SqlParseException, InterruptedException { SearchHits results = query(String.format("SELECT * FROM %s/location WHERE GEO_DISTANCE(center,'1km',100.5,0.500001)", TEST_INDEX_LOCATION)); - org.junit.Assert.assertEquals(1,results.getTotalHits()); + org.junit.Assert.assertEquals(1,results.getTotalHits().value); SearchHit result = results.getAt(0); Assert.assertEquals("square",result.getSourceAsMap().get("description")); } @@ -644,7 +644,7 @@ public void geoDistance() throws SQLFeatureNotSupportedException, SqlParseExcept @Test public void geoPolygon() throws SQLFeatureNotSupportedException, SqlParseException, InterruptedException { SearchHits results = query(String.format("SELECT * FROM %s/location WHERE GEO_POLYGON(center,100,0,100.5,2,101.0,0)", TEST_INDEX_LOCATION)); - org.junit.Assert.assertEquals(1,results.getTotalHits()); + org.junit.Assert.assertEquals(1,results.getTotalHits().value); SearchHit result = results.getAt(0); Assert.assertEquals("square",result.getSourceAsMap().get("description")); } @@ -652,19 +652,19 @@ public void geoPolygon() throws SQLFeatureNotSupportedException, SqlParseExcepti @Test public void escapedCharactersCheck() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT * FROM %s/gotCharacters where MATCH_PHRASE(nickname, 'Daenerys \"Stormborn\"') LIMIT 1000", TEST_INDEX_GAME_OF_THRONES)); - Assert.assertEquals(1, response.getTotalHits()); + Assert.assertEquals(1, response.getTotalHits().value); } @Test public void complexObjectSearch() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT * FROM %s/gotCharacters where match_phrase(name.firstname, 'Jaime') LIMIT 1000", TEST_INDEX_GAME_OF_THRONES)); - Assert.assertEquals(1, response.getTotalHits()); + Assert.assertEquals(1, response.getTotalHits().value); } @Test public void complexObjectReutrnField() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT parents.father FROM %s/gotCharacters where match_phrase(name.firstname, 'Brandon') LIMIT 1000", TEST_INDEX_GAME_OF_THRONES)); - Assert.assertEquals(1, response.getTotalHits()); + Assert.assertEquals(1, response.getTotalHits().value); Map sourceAsMap = response.getHits()[0].getSourceAsMap(); Assert.assertEquals("Eddard",((HashMap)sourceAsMap.get("parents")).get("father")); } @@ -687,7 +687,7 @@ public void complexObjectReutrnField() throws IOException, SqlParseException, SQ public void notLikeTests() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ //cant use string.format cause of %d SearchHits response = query("SELECT name FROM " +TEST_INDEX_GAME_OF_THRONES + "/gotCharacters where name.firstname not like '%d' and name is not null LIMIT 1000"); - Assert.assertEquals(3, response.getTotalHits()); + Assert.assertEquals(3, response.getTotalHits().value); for(SearchHit hit : response.getHits()) { Map sourceAsMap = hit.getSourceAsMap(); String name = ((HashMap) sourceAsMap.get("name")).get("firstname").toString(); @@ -698,13 +698,13 @@ public void notLikeTests() throws IOException, SqlParseException, SQLFeatureNotS @Test public void isNullTest() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query("SELECT name FROM " +TEST_INDEX_GAME_OF_THRONES + "/gotCharacters where nickname IS NULL LIMIT 1000"); - Assert.assertEquals(6, response.getTotalHits()); + Assert.assertEquals(6, response.getTotalHits().value); } @Test public void isNotNullTest() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query("SELECT name FROM " +TEST_INDEX_GAME_OF_THRONES + "/gotCharacters where nickname IS NOT NULL LIMIT 1000"); - Assert.assertEquals(1, response.getTotalHits()); + Assert.assertEquals(1, response.getTotalHits().value); } @@ -715,7 +715,7 @@ public void useScrollNoParams() throws IOException, SqlParseException, SQLFeatur SearchHits hits = response.getHits(); //default is 50 , es5.0 functionality now returns docs on first scroll Assert.assertEquals(50,hits.getHits().length); - Assert.assertEquals(1000,hits.getTotalHits()); + Assert.assertEquals(1000,hits.getTotalHits().value); } @Test @@ -724,7 +724,7 @@ public void useScrollWithParams() throws IOException, SqlParseException, SQLFeat Assert.assertNotNull(response.getScrollId()); SearchHits hits = response.getHits(); Assert.assertEquals(10,hits.getHits().length); - Assert.assertEquals(1000,hits.getTotalHits()); + Assert.assertEquals(1000,hits.getTotalHits().value); } @@ -734,7 +734,7 @@ public void useScrollWithOrderByAndParams() throws IOException, SqlParseExceptio Assert.assertNotNull(response.getScrollId()); SearchHits hits = response.getHits(); Assert.assertEquals(5,hits.getHits().length); - Assert.assertEquals(1000,hits.getTotalHits()); + Assert.assertEquals(1000,hits.getTotalHits().value); for(SearchHit hit : hits){ Assert.assertEquals(20,hit.getSourceAsMap().get("age")); } @@ -815,13 +815,13 @@ public void idsQuerySubQueryIds() throws SqlParseException, SQLFeatureNotSupport @Test public void nestedEqualsTestFieldNormalField() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT * FROM %s/nestedType where nested(message.info)='b'", TEST_INDEX_NESTED_TYPE)); - Assert.assertEquals(1, response.getTotalHits()); + Assert.assertEquals(1, response.getTotalHits().value); } @Test public void nestedEqualsTestFieldInsideArrays() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT * FROM %s/nestedType where nested(message.info) = 'a'", TEST_INDEX_NESTED_TYPE)); - Assert.assertEquals(2, response.getTotalHits()); + Assert.assertEquals(2, response.getTotalHits().value); } // @Test @@ -833,49 +833,49 @@ public void nestedEqualsTestFieldInsideArrays() throws IOException, SqlParseExce @Test public void complexNestedQueryBothOnSameObject() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT * FROM %s/nestedType where nested('message',message.info = 'a' and message.author ='i' ) ", TEST_INDEX_NESTED_TYPE)); - Assert.assertEquals(1, response.getTotalHits()); + Assert.assertEquals(1, response.getTotalHits().value); } @Test public void complexNestedQueryNotBothOnSameObject() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT * FROM %s/nestedType where nested('message',message.info = 'a' and message.author ='h' ) ", TEST_INDEX_NESTED_TYPE)); - Assert.assertEquals(0, response.getTotalHits()); + Assert.assertEquals(0, response.getTotalHits().value); } @Test public void nestedOnInTermsQuery() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT * FROM %s/nestedType where nested(message.info) = IN_TERMS(a,b)", TEST_INDEX_NESTED_TYPE)); - Assert.assertEquals(3, response.getTotalHits()); + Assert.assertEquals(3, response.getTotalHits().value); } @Test public void childrenEqualsTestFieldNormalField() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT * FROM %s/joinType where children(childrenType, info)='b'", TEST_INDEX_JOIN_TYPE)); - Assert.assertEquals(1, response.getTotalHits()); + Assert.assertEquals(1, response.getTotalHits().value); } @Test public void childrenOnInQuery() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT * FROM %s/joinType where children(childrenType, info) in ('a','b')", TEST_INDEX_JOIN_TYPE)); - Assert.assertEquals(2, response.getTotalHits()); + Assert.assertEquals(2, response.getTotalHits().value); } @Test public void complexChildrenQueryBothOnSameObject() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT * FROM %s/joinType where children(childrenType, info = 'a' and author ='e' ) ", TEST_INDEX_JOIN_TYPE)); - Assert.assertEquals(1, response.getTotalHits()); + Assert.assertEquals(1, response.getTotalHits().value); } @Test public void complexChildrenQueryNotBothOnSameObject() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT * FROM %s/joinType where children(childrenType, info = 'a' and author ='j' ) ", TEST_INDEX_JOIN_TYPE)); - Assert.assertEquals(0, response.getTotalHits()); + Assert.assertEquals(0, response.getTotalHits().value); } @Test public void childrenOnInTermsQuery() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT * FROM %s/joinType where children(childrenType, info) = IN_TERMS(a,b)", TEST_INDEX_JOIN_TYPE)); - Assert.assertEquals(2, response.getTotalHits()); + Assert.assertEquals(2, response.getTotalHits().value); } /** We are not going to support Hints as it is not part of standard SQL */ diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/SQLFunctionsTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/SQLFunctionsTest.java index 11284c2fea..dfe6a4c17e 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/SQLFunctionsTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/SQLFunctionsTest.java @@ -257,7 +257,7 @@ public void functionLogs() throws Exception { @Test public void functionPow() throws Exception { String query = "SELECT pow(account_number, 2) as key,"+ - "abs(age - 60) as new_age from " + TEST_INDEX_ACCOUNT + "/account limit 1"; + "abs(age - 60) as new_age from " + TEST_INDEX_ACCOUNT + "/account WHERE firstname = 'Virginia' and lastname='Ayala' limit 1"; CSVResult csvResult = getCsvResult(false, query); List content = csvResult.getLines(); Assert.assertTrue(content.toString().contains("625")); diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/TestsConstants.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/TestsConstants.java index fcebfee627..b34f3e1593 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/TestsConstants.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/TestsConstants.java @@ -41,5 +41,5 @@ public class TestsConstants { public final static String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; public final static String TS_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; - public final static String SIMPLE_DATE_FORMAT = "YYYY-MM-dd"; + public final static String SIMPLE_DATE_FORMAT = "yyyy-MM-dd"; } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/DateFunctionsTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/DateFunctionsTest.java index 7fdd4f891f..b3fb33e013 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/DateFunctionsTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/DateFunctionsTest.java @@ -48,7 +48,7 @@ public void yearInSelect() { assertTrue( scriptContainsString( scriptField, - "doc['creationDate'].date.year")); + "doc['creationDate'].value.year")); } @Test @@ -60,7 +60,7 @@ public void yearInWhere() { assertTrue( scriptContainsString( scriptFilter, - "doc['creationDate'].date.year")); + "doc['creationDate'].value.year")); assertTrue( scriptHasPattern( scriptFilter, @@ -75,7 +75,7 @@ public void weekOfYearInSelect() { assertTrue( scriptContainsString( scriptField, - "doc['creationDate'].date.weekOfWeekyear")); + "doc['creationDate'].value.weekOfWeekyear")); } @Test @@ -87,7 +87,7 @@ public void weekOfYearInWhere() { assertTrue( scriptContainsString( scriptFilter, - "doc['creationDate'].date.weekOfWeekyear")); + "doc['creationDate'].value.weekOfWeekyear")); assertTrue( scriptHasPattern( scriptFilter, @@ -102,7 +102,7 @@ public void dayOfMonth() { assertTrue( scriptContainsString( scriptField, - "doc['creationDate'].date.dayOfMonth")); + "doc['creationDate'].value.dayOfMonth")); } @Test @@ -113,7 +113,7 @@ public void hourOfDay() { assertTrue( scriptContainsString( scriptField, - "doc['creationDate'].date.hourOfDay")); + "doc['creationDate'].value.hourOfDay")); } @Test @@ -124,7 +124,7 @@ public void secondOfMinute() { assertTrue( scriptContainsString( scriptField, - "doc['creationDate'].date.secondOfMinute")); + "doc['creationDate'].value.secondOfMinute")); } } diff --git a/src/test/resources/join_objects.json b/src/test/resources/join_objects.json index ed142621fd..a1fdc86850 100644 --- a/src/test/resources/join_objects.json +++ b/src/test/resources/join_objects.json @@ -6,13 +6,13 @@ {"messageTile": "c", "join_field": "parentType"} {"index":{"_type": "joinType", "_id":"4"}} {"messageTile": "d", "join_field": "parentType"} -{"index":{"_type": "joinType", "_id":"5", "_routing": "1"}} +{"index":{"_type": "joinType", "_id":"5", "routing": "1"}} {"info":"a","author":"e","dayOfWeek":1, "join_field": {"name": "childrenType", "parent": "1"}} -{"index":{"_type": "joinType", "_id":"6", "_routing": "2"}} +{"index":{"_type": "joinType", "_id":"6", "routing": "2"}} {"info":"b","author":"f","dayOfWeek":2, "join_field": {"name": "childrenType", "parent": "2"}} -{"index":{"_type": "joinType", "_id":"7", "_routing": "3"}} +{"index":{"_type": "joinType", "_id":"7", "routing": "3"}} {"info":"c","author":"g","dayOfWeek":1, "join_field": {"name": "childrenType", "parent": "3"}} -{"index":{"_type": "joinType", "_id":"8", "_routing": "4"}} +{"index":{"_type": "joinType", "_id":"8", "routing": "4"}} {"info":"d","author":"h","dayOfWeek":3, "join_field": {"name": "childrenType", "parent": "4"}} -{"index":{"_type": "joinType", "_id":"9", "_routing": "1"}} +{"index":{"_type": "joinType", "_id":"9", "routing": "1"}} {"info":"a","author":"h","dayOfWeek":3, "join_field": {"name": "childrenType", "parent": "1"}} From eb14e265550dedf819767292b27aaac0967c43aa Mon Sep 17 00:00:00 2001 From: Peng Date: Tue, 14 May 2019 14:41:40 -0700 Subject: [PATCH 2/2] remove the unnecessary order by in integration test --- .../sql/intgtest/DateFormatTest.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/DateFormatTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/DateFormatTest.java index f62c858f15..c384794186 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/DateFormatTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/DateFormatTest.java @@ -66,9 +66,7 @@ public void lessThan() { @Test public void lessThanOrEqualTo() { assertThat( - query(SELECT_FROM + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') <= '2014-08-18' " + - "ORDER BY insert_time " + - "LIMIT 1000"), + query(SELECT_FROM + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') <= '2014-08-18' LIMIT 1000"), containsInAnyOrder("2014-08-17", "2014-08-18") ); } @@ -109,7 +107,6 @@ public void or() { query(SELECT_FROM + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') < '2014-08-18' " + "OR date_format(insert_time, 'yyyy-MM-dd', 'UTC') > '2014-08-23'" + - "ORDER BY insert_time " + "LIMIT 3000"), containsInAnyOrder("2014-08-17", "2014-08-24") );