From bc1ba1e39af34763d6dc98b9120b5b3d9a1f294f Mon Sep 17 00:00:00 2001 From: Huo Date: Fri, 10 May 2019 08:02:48 -0700 Subject: [PATCH] Support for Elasticsearch 7.0.1 (#47) --- build.gradle | 4 +- gradle.properties | 2 +- gradle/wrapper/gradle-wrapper.properties | 18 +----- .../sql/domain/Having.java | 2 +- .../sql/domain/SearchResult.java | 7 ++- .../sql/executor/csv/CSVResultsExtractor.java | 12 ++-- .../sql/executor/format/SelectResultSet.java | 3 +- .../executor/join/ElasticJoinExecutor.java | 12 ++-- .../sql/executor/join/ElasticUtils.java | 15 +++-- .../join/NestedLoopsElasticExecutor.java | 20 +++--- .../sql/executor/multi/MinusExecutor.java | 6 +- .../sql/executor/multi/UnionExecutor.java | 4 +- .../sql/query/maker/AggMaker.java | 13 ++-- .../sql/esintgtest/AggregationIT.java | 22 +++++-- .../esintgtest/CustomExternalTestCluster.java | 8 +-- .../sql/esintgtest/DateFormatIT.java | 18 +++--- .../sql/esintgtest/PreparedStatementIT.java | 1 - .../sql/esintgtest/SQLIntegTestCase.java | 3 +- .../sql/esintgtest/TestUtils.java | 2 +- .../sql/esintgtest/TestsConstants.java | 2 +- .../sql/intgtest/AggregationTest.java | 24 +++---- .../sql/intgtest/HashJoinTest.java | 2 +- .../sql/intgtest/HavingTest.java | 2 +- .../sql/intgtest/NestedFieldQueryTest.java | 6 +- .../sql/intgtest/PreparedStatementTest.java | 47 ++++---------- .../sql/intgtest/QueryTest.java | 4 +- .../sql/unittest/DateFormatTest.java | 2 +- .../sql/unittest/HavingTest.java | 2 +- .../executor/join/ElasticUtilsTest.java | 62 +++++++++++++++++++ .../unittest/planner/QueryPlannerTest.java | 6 +- 30 files changed, 189 insertions(+), 142 deletions(-) create mode 100644 src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/executor/join/ElasticUtilsTest.java diff --git a/build.gradle b/build.gradle index 563f8bf922..e30fcf01d8 100644 --- a/build.gradle +++ b/build.gradle @@ -16,7 +16,7 @@ buildscript { ext { - es_version = System.getProperty("es.version", "6.7.1") + es_version = System.getProperty("es.version", "7.0.1") } // This isn't applying from repositories.gradle so repeating it here repositories { @@ -95,7 +95,7 @@ compileTestJava { } // TODO: Need to update integration test to use ElasticSearch test framework -unitTest { +test { include '**/*Test.class' exclude 'com/amazon/opendistroforelasticsearch/sql/intgtest/**' // Gradle runs unit test using a working directory other and project root diff --git a/gradle.properties b/gradle.properties index 0db2a19844..0243e9e7a9 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,4 +13,4 @@ # permissions and limitations under the License. # -version=0.7 +version=1.0.0 diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 7abc42175f..d308c1ae30 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,20 +1,6 @@ -# -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the "license" file accompanying this file. This file is distributed -# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -# express or implied. See the License for the specific language governing -# permissions and limitations under the License. -# - +#Thu May 02 16:03:13 PDT 2019 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-5.3-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-5.3-all.zip diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Having.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Having.java index 5ab582e413..08d999d321 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Having.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/Having.java @@ -29,7 +29,7 @@ import java.util.Map; import static java.util.stream.Collectors.joining; -import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.bucketSelector; +import static org.elasticsearch.search.aggregations.PipelineAggregatorBuilders.bucketSelector; /** * Domain object for HAVING clause in SQL which covers both the parsing and explain logic. diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/SearchResult.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/SearchResult.java index 21fe67ce88..dc43bc527b 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/SearchResult.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/domain/SearchResult.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; import org.elasticsearch.action.search.SearchResponse; @@ -34,8 +35,8 @@ import org.elasticsearch.search.aggregations.bucket.terms.LongTerms; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggregation; -import org.elasticsearch.search.aggregations.metrics.tophits.InternalTopHits; -import org.elasticsearch.search.aggregations.metrics.valuecount.InternalValueCount; +import org.elasticsearch.search.aggregations.metrics.InternalTopHits; +import org.elasticsearch.search.aggregations.metrics.InternalValueCount; public class SearchResult { /** @@ -49,7 +50,7 @@ public class SearchResult { public SearchResult(SearchResponse resp) { SearchHits hits = resp.getHits(); - this.total = hits.getTotalHits(); + this.total = Optional.ofNullable(hits.getTotalHits()).map(totalHits -> totalHits.value).orElse(0L); results = new ArrayList<>(hits.getHits().length); for (SearchHit searchHit : hits.getHits()) { if (searchHit.getSourceAsMap() != null) { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/csv/CSVResultsExtractor.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/csv/CSVResultsExtractor.java index ca84ce4727..a62dc9b0b9 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/csv/CSVResultsExtractor.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/csv/CSVResultsExtractor.java @@ -24,12 +24,12 @@ import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregation; import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation; -import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBounds; -import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile; -import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles; -import org.elasticsearch.search.aggregations.metrics.stats.Stats; -import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats; -import org.elasticsearch.search.aggregations.metrics.tophits.TopHits; +import org.elasticsearch.search.aggregations.metrics.GeoBounds; +import org.elasticsearch.search.aggregations.metrics.Percentile; +import org.elasticsearch.search.aggregations.metrics.Percentiles; +import org.elasticsearch.search.aggregations.metrics.Stats; +import org.elasticsearch.search.aggregations.metrics.ExtendedStats; +import org.elasticsearch.search.aggregations.metrics.TopHits; import com.amazon.opendistroforelasticsearch.sql.utils.Util; import java.util.*; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java index c13472c5c9..d2b247e72e 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java @@ -38,6 +38,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import static org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData; @@ -430,7 +431,7 @@ private void extractData() { SearchHits searchHits = (SearchHits) queryResult; this.size = searchHits.getHits().length; - this.totalHits = searchHits.totalHits; + this.totalHits = Optional.ofNullable(searchHits.getTotalHits()).map(th -> th.value).orElse(0L); this.rows = populateRows(searchHits); } else if (queryResult instanceof Aggregations) { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/ElasticJoinExecutor.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/ElasticJoinExecutor.java index ed8267040e..d2b1591c49 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/ElasticJoinExecutor.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/ElasticJoinExecutor.java @@ -15,13 +15,19 @@ package com.amazon.opendistroforelasticsearch.sql.executor.join; +import com.amazon.opendistroforelasticsearch.sql.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; import com.amazon.opendistroforelasticsearch.sql.executor.ElasticHitsExecutor; +import com.amazon.opendistroforelasticsearch.sql.query.SqlElasticRequestBuilder; import com.amazon.opendistroforelasticsearch.sql.query.join.HashJoinElasticRequestBuilder; import com.amazon.opendistroforelasticsearch.sql.query.join.JoinRequestBuilder; import com.amazon.opendistroforelasticsearch.sql.query.join.NestedLoopsElasticRequestBuilder; import com.amazon.opendistroforelasticsearch.sql.query.join.TableInJoinRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.query.planner.HashJoinQueryPlanRequestBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.search.TotalHits; +import org.apache.lucene.search.TotalHits.Relation; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; @@ -35,10 +41,6 @@ import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortOrder; -import com.amazon.opendistroforelasticsearch.sql.domain.Field; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.query.SqlElasticRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.planner.HashJoinQueryPlanRequestBuilder; import java.io.IOException; import java.util.Collection; @@ -102,7 +104,7 @@ public void run() throws IOException, SqlParseException { protected abstract List innerRun() throws IOException, SqlParseException ; public SearchHits getHits(){ - return new SearchHits(results.toArray(new SearchHit[results.size()]), results.size(), 1.0f); + return new SearchHits(results.toArray(new SearchHit[results.size()]), new TotalHits(results.size(), Relation.EQUAL_TO), 1.0f); } public static ElasticJoinExecutor createJoinExecutor(Client client, SqlElasticRequestBuilder requestBuilder){ diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/ElasticUtils.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/ElasticUtils.java index 0a30b911de..26b9a48c65 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/ElasticUtils.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/ElasticUtils.java @@ -15,7 +15,10 @@ package com.amazon.opendistroforelasticsearch.sql.executor.join; +import com.amazon.opendistroforelasticsearch.sql.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.query.join.BackOffRetryStrategy; import com.google.common.collect.ImmutableMap; +import org.apache.lucene.search.TotalHits.Relation; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; @@ -30,12 +33,11 @@ import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortOrder; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.query.join.BackOffRetryStrategy; import java.io.IOException; import java.util.HashMap; import java.util.List; +import java.util.Optional; import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS; @@ -60,11 +62,11 @@ public static SearchResponse scrollOneTimeWithHits(Client client, SearchRequestB } - //use our deserializer instead of results toXcontent because the source field is differnet from sourceAsMap. + //use our deserializer instead of results toXcontent because the source field is different from sourceAsMap. public static String hitsAsStringResult(SearchHits results, MetaSearchResult metaResults) throws IOException { if(results == null) return null; Object[] searchHits; - searchHits = new Object[(int) results.getTotalHits()]; + searchHits = new Object[Optional.ofNullable(results.getTotalHits()).map(th -> th.value).orElse(0L).intValue()]; int i = 0; for(SearchHit hit : results) { HashMap value = new HashMap<>(); @@ -76,7 +78,10 @@ public static String hitsAsStringResult(SearchHits results, MetaSearchResult met i++; } HashMap hits = new HashMap<>(); - hits.put("total",results.getTotalHits()); + hits.put("total", ImmutableMap.of( + "value", Optional.ofNullable(results.getTotalHits()).map(th -> th.value).orElse(0L), + "relation", Optional.ofNullable(results.getTotalHits()).map(th -> th.relation).orElse(Relation.EQUAL_TO) + )); hits.put("max_score",results.getMaxScore()); hits.put("hits",searchHits); XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint(); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/NestedLoopsElasticExecutor.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/NestedLoopsElasticExecutor.java index 9af7a684bd..a74fa8ebec 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/NestedLoopsElasticExecutor.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/join/NestedLoopsElasticExecutor.java @@ -16,7 +16,16 @@ package com.amazon.opendistroforelasticsearch.sql.executor.join; import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource; +import com.amazon.opendistroforelasticsearch.sql.domain.Condition; +import com.amazon.opendistroforelasticsearch.sql.domain.Select; +import com.amazon.opendistroforelasticsearch.sql.domain.Where; import com.amazon.opendistroforelasticsearch.sql.esdomain.ESClient; +import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.query.DefaultQueryAction; +import com.amazon.opendistroforelasticsearch.sql.query.join.BackOffRetryStrategy; +import com.amazon.opendistroforelasticsearch.sql.query.join.NestedLoopsElasticRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.query.join.TableInJoinRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.query.maker.Maker; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.search.MultiSearchRequest; @@ -28,15 +37,6 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; -import com.amazon.opendistroforelasticsearch.sql.domain.Condition; -import com.amazon.opendistroforelasticsearch.sql.domain.Select; -import com.amazon.opendistroforelasticsearch.sql.domain.Where; -import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; -import com.amazon.opendistroforelasticsearch.sql.query.DefaultQueryAction; -import com.amazon.opendistroforelasticsearch.sql.query.join.BackOffRetryStrategy; -import com.amazon.opendistroforelasticsearch.sql.query.join.NestedLoopsElasticRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.join.TableInJoinRequestBuilder; -import com.amazon.opendistroforelasticsearch.sql.query.maker.Maker; import java.util.ArrayList; import java.util.List; @@ -224,7 +224,7 @@ private FetchWithScrollResponse firstFetch(TableInJoinRequestBuilder tableReques else { //scroll request with max. responseWithHits = scrollOneTimeWithMax(client,tableRequest); - if(responseWithHits.getHits().getTotalHits() < MAX_RESULTS_ON_ONE_FETCH) + if(responseWithHits.getHits().getTotalHits() != null && responseWithHits.getHits().getTotalHits().value < MAX_RESULTS_ON_ONE_FETCH) needScrollForFirstTable = true; } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/MinusExecutor.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/MinusExecutor.java index 6cc27f768d..bfac82fca2 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/MinusExecutor.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/MinusExecutor.java @@ -17,6 +17,8 @@ import com.amazon.opendistroforelasticsearch.sql.executor.ElasticHitsExecutor; import com.amazon.opendistroforelasticsearch.sql.executor.join.ElasticUtils; +import org.apache.lucene.search.TotalHits; +import org.apache.lucene.search.TotalHits.Relation; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.document.DocumentField; @@ -136,7 +138,7 @@ private void fillMinusHitsFromOneField(String fieldName, Set fieldValues } int totalSize = currentId - 1; SearchHit[] unionHitsArr = minusHitsList.toArray(new SearchHit[totalSize]); - this.minusHits = new SearchHits(unionHitsArr, totalSize,1.0f); + this.minusHits = new SearchHits(unionHitsArr, new TotalHits(totalSize, Relation.EQUAL_TO), 1.0f); } private void fillMinusHitsFromResults(Set comperableHitResults) { @@ -164,7 +166,7 @@ private void fillMinusHitsFromResults(Set comperableHitResu } int totalSize = currentId - 1; SearchHit[] unionHitsArr = minusHitsList.toArray(new SearchHit[totalSize]); - this.minusHits = new SearchHits(unionHitsArr, totalSize,1.0f); + this.minusHits = new SearchHits(unionHitsArr, new TotalHits(totalSize, Relation.EQUAL_TO), 1.0f); } private Set runWithScrollings() { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/UnionExecutor.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/UnionExecutor.java index 3b57cb9cce..683ea690a0 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/UnionExecutor.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/multi/UnionExecutor.java @@ -16,6 +16,8 @@ package com.amazon.opendistroforelasticsearch.sql.executor.multi; import com.amazon.opendistroforelasticsearch.sql.executor.ElasticHitsExecutor; +import org.apache.lucene.search.TotalHits; +import org.apache.lucene.search.TotalHits.Relation; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.text.Text; @@ -54,7 +56,7 @@ public void run() throws IOException, SqlParseException { fillInternalSearchHits(unionHits,secondResponse.getHits().getHits(),this.multiQueryBuilder.getSecondTableFieldToAlias()); int totalSize = unionHits.size(); SearchHit[] unionHitsArr = unionHits.toArray(new SearchHit[totalSize]); - this.results = new SearchHits(unionHitsArr, totalSize,1.0f); + this.results = new SearchHits(unionHitsArr, new TotalHits(totalSize, Relation.EQUAL_TO),1.0f); } private void fillInternalSearchHits(List unionHits, SearchHit[] hits, Map fieldNameToAlias) { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/maker/AggMaker.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/maker/AggMaker.java index eb2985cc37..c441f254f9 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/maker/AggMaker.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/maker/AggMaker.java @@ -49,13 +49,12 @@ import org.elasticsearch.search.aggregations.bucket.range.DateRangeAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBoundsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.scripted.ScriptedMetricAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.PercentilesAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.ScriptedMetricAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.sort.SortOrder; -import org.joda.time.DateTimeZone; import com.amazon.opendistroforelasticsearch.sql.utils.Util; import com.amazon.opendistroforelasticsearch.sql.parser.ChildrenType; import com.amazon.opendistroforelasticsearch.sql.parser.NestedType; @@ -473,7 +472,7 @@ private ValuesSourceAggregationBuilder dateRange(MethodField field) { dateRange.format(value); continue; } else if ("time_zone".equals(kv.key)) { - dateRange.timeZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone(ZoneOffset.of(value)))); + dateRange.timeZone(ZoneOffset.of(value)); continue; } else if ("from".equals(kv.key)) { dateRange.addUnboundedFrom(kv.value.toString()); @@ -523,7 +522,7 @@ private DateHistogramAggregationBuilder dateHistogram(MethodField field) throws dateHistogram.format(value); break; case "time_zone": - dateHistogram.timeZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone(ZoneOffset.of(value)))); + dateHistogram.timeZone(ZoneOffset.of(value)); break; case "min_doc_count": dateHistogram.minDocCount(Long.parseLong(value)); diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/AggregationIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/AggregationIT.java index 456ba64216..b13f432c7b 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/AggregationIT.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/AggregationIT.java @@ -443,11 +443,13 @@ public void topHitTest() throws IOException { final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); - Assert.assertThat(gender.query(maleBucketPrefix + "/topHits(size=3,age=desc)/hits/total"),equalTo(507)); + Assert.assertThat(gender.query(maleBucketPrefix + "/topHits(size=3,age=desc)/hits/total/value"),equalTo(507)); + Assert.assertThat(gender.query(maleBucketPrefix + "/topHits(size=3,age=desc)/hits/total/relation"),equalTo("eq")); Assert.assertThat(((JSONArray)gender.query(maleBucketPrefix + "/topHits(size=3,age=desc)/hits/hits")).length(), equalTo(3)); Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); - Assert.assertThat(gender.query(femaleBucketPrefix + "/topHits(size=3,age=desc)/hits/total"), equalTo(493)); + Assert.assertThat(gender.query(femaleBucketPrefix + "/topHits(size=3,age=desc)/hits/total/value"), equalTo(493)); + Assert.assertThat(gender.query(femaleBucketPrefix + "/topHits(size=3,age=desc)/hits/total/relation"), equalTo("eq")); Assert.assertThat(((JSONArray)gender.query(femaleBucketPrefix + "/topHits(size=3,age=desc)/hits/hits")).length(), equalTo(3)); } @@ -469,15 +471,19 @@ public void topHitTest_WithInclude() throws IOException { final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); - Assert.assertThat(gender.query(maleBucketPrefix + "/topHits(size=3,age=desc,include=age)/hits/total"), + Assert.assertThat(gender.query(maleBucketPrefix + "/topHits(size=3,age=desc,include=age)/hits/total/value"), equalTo(507)); + Assert.assertThat(gender.query(maleBucketPrefix + "/topHits(size=3,age=desc,include=age)/hits/total/relation"), + equalTo("eq")); Assert.assertThat(((JSONArray)gender.query( maleBucketPrefix + "/topHits(size=3,age=desc,include=age)/hits/hits")).length(), equalTo(3)); Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); - Assert.assertThat(gender.query(femaleBucketPrefix + "/topHits(size=3,age=desc,include=age)/hits/total"), + Assert.assertThat(gender.query(femaleBucketPrefix + "/topHits(size=3,age=desc,include=age)/hits/total/value"), equalTo(493)); + Assert.assertThat(gender.query(femaleBucketPrefix + "/topHits(size=3,age=desc,include=age)/hits/total/relation"), + equalTo("eq")); Assert.assertThat(((JSONArray)gender.query( femaleBucketPrefix + "/topHits(size=3,age=desc,include=age)/hits/hits")).length(), equalTo(3)); @@ -533,15 +539,19 @@ public void topHitTest_WithExclude() throws IOException { final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); - Assert.assertThat(gender.query(maleBucketPrefix + "/topHits(size=3,exclude=lastname,age=desc)/hits/total"), + Assert.assertThat(gender.query(maleBucketPrefix + "/topHits(size=3,exclude=lastname,age=desc)/hits/total/value"), equalTo(507)); + Assert.assertThat(gender.query(maleBucketPrefix + "/topHits(size=3,exclude=lastname,age=desc)/hits/total/relation"), + equalTo("eq")); Assert.assertThat(((JSONArray)gender.query( maleBucketPrefix + "/topHits(size=3,exclude=lastname,age=desc)/hits/hits")).length(), equalTo(3)); Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); - Assert.assertThat(gender.query(femaleBucketPrefix + "/topHits(size=3,exclude=lastname,age=desc)/hits/total"), + Assert.assertThat(gender.query(femaleBucketPrefix + "/topHits(size=3,exclude=lastname,age=desc)/hits/total/value"), equalTo(493)); + Assert.assertThat(gender.query(femaleBucketPrefix + "/topHits(size=3,exclude=lastname,age=desc)/hits/total/relation"), + equalTo("eq")); Assert.assertThat(((JSONArray)gender.query( femaleBucketPrefix + "/topHits(size=3,exclude=lastname,age=desc)/hits/hits")).length(), equalTo(3)); diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/CustomExternalTestCluster.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/CustomExternalTestCluster.java index 590d9ec5f0..981be35e76 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/CustomExternalTestCluster.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/CustomExternalTestCluster.java @@ -29,8 +29,8 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.TestCluster; -import org.elasticsearch.transport.MockTcpTransportPlugin; import org.elasticsearch.transport.MockTransportClient; +import org.elasticsearch.transport.nio.MockNioTransportPlugin; import java.io.IOException; import java.net.InetSocketAddress; @@ -71,10 +71,10 @@ public CustomExternalTestCluster(Path tempDir, Settings additionalSettings, Coll if (addMockTcpTransport) { String transport = getTestTransportType(); clientSettingsBuilder.put(NetworkModule.TRANSPORT_TYPE_KEY, transport); - if (pluginClasses.contains(MockTcpTransportPlugin.class) == false) { + if (pluginClasses.contains(MockNioTransportPlugin.class) == false) { pluginClasses = new ArrayList<>(pluginClasses); - if (transport.equals(MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME)) { - pluginClasses.add(MockTcpTransportPlugin.class); + if (transport.equals(MockNioTransportPlugin.MOCK_NIO_TRANSPORT_NAME)) { + pluginClasses.add(MockNioTransportPlugin.class); } } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/DateFormatIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/DateFormatIT.java index 23f907025f..adc88edfc0 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/DateFormatIT.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/DateFormatIT.java @@ -53,7 +53,7 @@ protected void init() throws Exception { @Test public void equalTo() throws SqlParseException { assertThat( - dateQuery(SELECT_FROM + "WHERE date_format(insert_time, 'YYYY-MM-dd', 'UTC') = '2014-08-17'"), + dateQuery(SELECT_FROM + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') = '2014-08-17'"), contains("2014-08-17") ); } @@ -61,7 +61,7 @@ public void equalTo() throws SqlParseException { @Test public void lessThan() throws SqlParseException { assertThat( - dateQuery(SELECT_FROM + "WHERE date_format(insert_time, 'YYYY-MM-dd', 'UTC') < '2014-08-18'"), + dateQuery(SELECT_FROM + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') < '2014-08-18'"), contains("2014-08-17") ); } @@ -69,7 +69,7 @@ public void lessThan() throws SqlParseException { @Test public void lessThanOrEqualTo() throws SqlParseException { assertThat( - dateQuery(SELECT_FROM + "WHERE date_format(insert_time, 'YYYY-MM-dd', 'UTC') <= '2014-08-18' " + + dateQuery(SELECT_FROM + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') <= '2014-08-18' " + "ORDER BY insert_time " + "LIMIT 1000"), contains("2014-08-17", "2014-08-18") @@ -79,7 +79,7 @@ public void lessThanOrEqualTo() throws SqlParseException { @Test public void greaterThan() throws SqlParseException { assertThat( - dateQuery(SELECT_FROM + "WHERE date_format(insert_time, 'YYYY-MM-dd', 'UTC') > '2014-08-23'"), + dateQuery(SELECT_FROM + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') > '2014-08-23'"), contains("2014-08-24") ); } @@ -87,7 +87,7 @@ public void greaterThan() throws SqlParseException { @Test public void greaterThanOrEqualTo() throws SqlParseException { assertThat( - dateQuery(SELECT_FROM + "WHERE date_format(insert_time, 'YYYY-MM-dd', 'UTC') >= '2014-08-23' " + + dateQuery(SELECT_FROM + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') >= '2014-08-23' " + "ORDER BY insert_time " + "LIMIT 2000"), contains("2014-08-23", "2014-08-24") @@ -98,8 +98,8 @@ public void greaterThanOrEqualTo() throws SqlParseException { public void and() throws SqlParseException{ assertThat( dateQuery(SELECT_FROM + - "WHERE date_format(insert_time, 'YYYY-MM-dd', 'UTC') >= '2014-08-21' " + - "AND date_format(insert_time, 'YYYY-MM-dd', 'UTC') <= '2014-08-23' " + + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') >= '2014-08-21' " + + "AND date_format(insert_time, 'yyyy-MM-dd', 'UTC') <= '2014-08-23' " + "ORDER BY insert_time " + "LIMIT 3000"), contains("2014-08-21", "2014-08-22", "2014-08-23") @@ -110,8 +110,8 @@ public void and() throws SqlParseException{ public void or() throws SqlParseException { assertThat( dateQuery(SELECT_FROM + - "WHERE date_format(insert_time, 'YYYY-MM-dd', 'UTC') < '2014-08-18' " + - "OR date_format(insert_time, 'YYYY-MM-dd', 'UTC') > '2014-08-23' " + + "WHERE date_format(insert_time, 'yyyy-MM-dd', 'UTC') < '2014-08-18' " + + "OR date_format(insert_time, 'yyyy-MM-dd', 'UTC') > '2014-08-23' " + "ORDER BY insert_time " + "LIMIT 1000"), contains("2014-08-17", "2014-08-24") diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/PreparedStatementIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/PreparedStatementIT.java index 990c85f7cd..99f001efc1 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/PreparedStatementIT.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/PreparedStatementIT.java @@ -39,7 +39,6 @@ protected void init() throws Exception { public void sample_test() { Client client = ESIntegTestCase.client(); SearchResponse response = client.prepareSearch(TestsConstants.TEST_INDEX_ACCOUNT).get(); - System.out.println(response.getHits().totalHits); } @Test diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java index 262979f98d..f711c60c13 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java @@ -195,8 +195,9 @@ protected JSONArray getHits(JSONObject response) { protected int getTotalHits(JSONObject response) { Assert.assertTrue(response.getJSONObject("hits").has("total")); + Assert.assertTrue(response.getJSONObject("hits").getJSONObject("total").has("value")); - return response.getJSONObject("hits").getInt("total"); + return response.getJSONObject("hits").getJSONObject("total").getInt("value"); } protected JSONObject getSource(JSONObject hit) { diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestUtils.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestUtils.java index cd81c6803b..1034c052d4 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestUtils.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestUtils.java @@ -136,7 +136,7 @@ public static String getOdbcIndexMapping() { "\t\t\"properties\":{\n" + "\t\t\t\"odbc_time\":{\n" + "\t\t\t\t\"type\":\"date\",\n" + - "\t\t\t\t\"format\": \"{'ts' ''yyyy-MM-dd HH:mm:ss.SSS''}\"\n" + + "\t\t\t\t\"format\": \"'{ts' ''yyyy-MM-dd HH:mm:ss.SSS'''}'\"\n" + "\t\t\t},\n" + "\t\t\t\"docCount\":{\n" + "\t\t\t\t\"type\":\"text\"\n" + diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestsConstants.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestsConstants.java index db782e9fc9..8ab3bc5b2a 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestsConstants.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestsConstants.java @@ -40,5 +40,5 @@ public class TestsConstants { public final static String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; public final static String TS_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; - public final static String SIMPLE_DATE_FORMAT = "YYYY-MM-dd"; + public final static String SIMPLE_DATE_FORMAT = "yyyy-MM-dd"; } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/AggregationTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/AggregationTest.java index f239af3ad9..fc477af163 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/AggregationTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/AggregationTest.java @@ -31,17 +31,17 @@ import org.elasticsearch.search.aggregations.bucket.nested.InternalNested; import org.elasticsearch.search.aggregations.bucket.nested.InternalReverseNested; import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.metrics.avg.Avg; -import org.elasticsearch.search.aggregations.metrics.geobounds.InternalGeoBounds; -import org.elasticsearch.search.aggregations.metrics.max.Max; -import org.elasticsearch.search.aggregations.metrics.min.Min; -import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles; -import org.elasticsearch.search.aggregations.metrics.stats.Stats; -import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats; -import org.elasticsearch.search.aggregations.metrics.sum.InternalSum; -import org.elasticsearch.search.aggregations.metrics.sum.Sum; -import org.elasticsearch.search.aggregations.metrics.tophits.InternalTopHits; -import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCount; +import org.elasticsearch.search.aggregations.metrics.Avg; +import org.elasticsearch.search.aggregations.metrics.InternalGeoBounds; +import org.elasticsearch.search.aggregations.metrics.Max; +import org.elasticsearch.search.aggregations.metrics.Min; +import org.elasticsearch.search.aggregations.metrics.Percentiles; +import org.elasticsearch.search.aggregations.metrics.Stats; +import org.elasticsearch.search.aggregations.metrics.ExtendedStats; +import org.elasticsearch.search.aggregations.metrics.InternalSum; +import org.elasticsearch.search.aggregations.metrics.Sum; +import org.elasticsearch.search.aggregations.metrics.InternalTopHits; +import org.elasticsearch.search.aggregations.metrics.ValueCount; import org.junit.Assert; import org.junit.Test; import com.amazon.opendistroforelasticsearch.sql.plugin.SearchDao; @@ -230,7 +230,7 @@ else if(key.equalsIgnoreCase("f")) { public void postFilterTest() throws Exception { SqlElasticSearchRequestBuilder select = getSearchRequestBuilder(String.format("SELECT /*! POST_FILTER({\"term\":{\"gender\":\"m\"}}) */ COUNT(*) FROM %s/account GROUP BY gender", TEST_INDEX_ACCOUNT)); SearchResponse res = (SearchResponse) select.get(); - Assert.assertEquals(507, res.getHits().totalHits); + Assert.assertEquals(507, res.getHits().getTotalHits().value); Aggregations result = res.getAggregations(); Terms gender = result.get("gender"); diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/HashJoinTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/HashJoinTest.java index aac7b13176..407c5efa66 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/HashJoinTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/HashJoinTest.java @@ -70,7 +70,7 @@ protected void assertEquals(SearchHits expected, SearchHits actual) { Collection expectedSet = wrapAndIgnoreUninterestedFields(expected.getHits()); Collection actualSet = wrapAndIgnoreUninterestedFields(actual.getHits()); try { - Assert.assertEquals(expected.totalHits, actual.totalHits); + Assert.assertEquals(expected.getTotalHits().value, actual.getTotalHits().value); Assert.assertEquals(expected.getHits().length, actual.getHits().length); Assert.assertEquals(expectedSet, actualSet); } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/HavingTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/HavingTest.java index b75f03bb6a..69d5c42c09 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/HavingTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/HavingTest.java @@ -21,7 +21,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCount; +import org.elasticsearch.search.aggregations.metrics.ValueCount; import org.hamcrest.Matcher; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/NestedFieldQueryTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/NestedFieldQueryTest.java index 9df3db1def..1cf87b69e9 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/NestedFieldQueryTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/NestedFieldQueryTest.java @@ -24,9 +24,9 @@ import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.bucket.nested.InternalNested; import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.metrics.avg.Avg; -import org.elasticsearch.search.aggregations.metrics.sum.Sum; -import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCount; +import org.elasticsearch.search.aggregations.metrics.Avg; +import org.elasticsearch.search.aggregations.metrics.Sum; +import org.elasticsearch.search.aggregations.metrics.ValueCount; import org.hamcrest.FeatureMatcher; import org.hamcrest.Matcher; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/PreparedStatementTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/PreparedStatementTest.java index b0533d23bb..b12a4488a4 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/PreparedStatementTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/PreparedStatementTest.java @@ -16,16 +16,16 @@ package com.amazon.opendistroforelasticsearch.sql.intgtest; import com.alibaba.druid.sql.parser.ParserException; -import com.amazon.opendistroforelasticsearch.sql.plugin.SearchDao; -import com.amazon.opendistroforelasticsearch.sql.request.SqlRequest; -import com.amazon.opendistroforelasticsearch.sql.request.SqlRequestFactory; import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; +import com.amazon.opendistroforelasticsearch.sql.plugin.SearchDao; import com.amazon.opendistroforelasticsearch.sql.query.QueryAction; import com.amazon.opendistroforelasticsearch.sql.query.SqlElasticSearchRequestBuilder; +import com.amazon.opendistroforelasticsearch.sql.request.SqlRequest; +import com.amazon.opendistroforelasticsearch.sql.request.SqlRequestFactory; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.http.HttpChannel; +import org.elasticsearch.http.HttpRequest; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -33,11 +33,12 @@ import org.junit.Test; import java.sql.SQLFeatureNotSupportedException; -import java.util.HashMap; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.greaterThan; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class PreparedStatementTest { @@ -76,7 +77,10 @@ public void search() { private SearchHits query(String request) { try { - SqlRequest sqlRequest = SqlRequestFactory.getSqlRequest(new TestRestRequest(request)); + final HttpRequest httpRequest = mock(HttpRequest.class); + when (httpRequest.uri()).thenReturn(""); + + SqlRequest sqlRequest = SqlRequestFactory.getSqlRequest(RestRequest.request(NamedXContentRegistry.EMPTY, httpRequest, mock(HttpChannel.class))); JSONObject jsonRequest = new JSONObject(request); String sql = sqlRequest.getSql(); @@ -93,33 +97,4 @@ private SearchHits query(String request) { throw new ParserException("Illegal sql expr in request: " + request, e); } } - - static class TestRestRequest extends RestRequest { - private String payload; - - TestRestRequest(String payload) { - super(NamedXContentRegistry.EMPTY, "", new HashMap<>()); - this.payload = payload; - } - - @Override - public Method method() { - return Method.POST; - } - - @Override - public String uri() { - return "uri"; - } - - @Override - public boolean hasContent() { - return true; - } - - @Override - public BytesReference innerContent() { - return new BytesArray(this.payload); - } - } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/QueryTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/QueryTest.java index cd0e660e5e..f94112dabf 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/QueryTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/intgtest/QueryTest.java @@ -61,7 +61,7 @@ public void multipleFromTest() throws IOException, SqlParseException, SQLFeature @Test public void indexWithWildcardTest() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT * FROM %s* LIMIT 1000", TEST_INDEX_BANK)); - assertThat(response.getTotalHits(), greaterThan(0L)); + assertThat(response.getTotalHits().value, greaterThan(0L)); } @@ -888,7 +888,7 @@ public void childrenOnInTermsQuery() throws IOException, SqlParseException, SQLF @Test(expected=IndexNotFoundException.class) public void multipleIndicesOneNotExistWithoutHint() throws IOException, SqlParseException, SQLFeatureNotSupportedException{ SearchHits response = query(String.format("SELECT * FROM %s,%s ", TEST_INDEX_ACCOUNT,"badindex")); - Assert.assertTrue(response.getTotalHits() > 0); + Assert.assertTrue(response.getTotalHits().value > 0); } @Test diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/DateFormatTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/DateFormatTest.java index e678b4eeb1..e5a8075f8f 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/DateFormatTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/DateFormatTest.java @@ -93,7 +93,7 @@ public void timeZoneCondition() { List q = query(SELECT_CNT_FROM_DATE + "WHERE date_format(creationDate, 'YYYY-MM-dd', 'America/Phoenix') > '2018-04-02'"); // Used hasProperty here as getter followed convention for obtaining ID and Feature Matcher was having issues with generic type to obtain value - assertThat(q, hasQueryWithValue("timeZone", hasProperty("ID", equalTo("America/Phoenix")))); + assertThat(q, hasQueryWithValue("timeZone", hasProperty("id", equalTo("America/Phoenix")))); } private List query(String sql) { diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/HavingTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/HavingTest.java index 0c9f382076..9e436b3d97 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/HavingTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/HavingTest.java @@ -27,7 +27,7 @@ import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.PipelineAggregationBuilder; -import org.elasticsearch.search.aggregations.pipeline.bucketselector.BucketSelectorPipelineAggregationBuilder; +import org.elasticsearch.search.aggregations.pipeline.BucketSelectorPipelineAggregationBuilder; import org.hamcrest.Matcher; import org.hamcrest.Matchers; import org.junit.Test; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/executor/join/ElasticUtilsTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/executor/join/ElasticUtilsTest.java new file mode 100644 index 0000000000..1bda31e0ce --- /dev/null +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/executor/join/ElasticUtilsTest.java @@ -0,0 +1,62 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.unittest.executor.join; + +import com.amazon.opendistroforelasticsearch.sql.executor.join.ElasticUtils; +import com.amazon.opendistroforelasticsearch.sql.executor.join.MetaSearchResult; +import org.apache.lucene.search.TotalHits; +import org.apache.lucene.search.TotalHits.Relation; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.json.JSONObject; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +import java.io.IOException; + +@RunWith(MockitoJUnitRunner.class) +public class ElasticUtilsTest { + + @Mock + MetaSearchResult metaSearchResult; + + /** + * test handling {@link TotalHits} correctly. + */ + @Test + public void hitsAsStringResult() throws IOException { + final SearchHits searchHits = new SearchHits(new SearchHit[]{}, new TotalHits(1, Relation.EQUAL_TO), 0); + final String result = ElasticUtils.hitsAsStringResult(searchHits, metaSearchResult); + + Assert.assertEquals(1, new JSONObject(result).query("/hits/total/value")); + Assert.assertEquals(Relation.EQUAL_TO.toString(), new JSONObject(result).query("/hits/total/relation")); + } + + /** + * test handling {@link TotalHits} with null value correctly. + */ + @Test + public void test_hitsAsStringResult_withNullTotalHits() throws IOException { + final SearchHits searchHits = new SearchHits(new SearchHit[]{}, null, 0); + final String result = ElasticUtils.hitsAsStringResult(searchHits, metaSearchResult); + + Assert.assertEquals(0, new JSONObject(result).query("/hits/total/value")); + Assert.assertEquals(Relation.EQUAL_TO.toString(), new JSONObject(result).query("/hits/total/relation")); + } +} \ No newline at end of file diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/QueryPlannerTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/QueryPlannerTest.java index eea6b0fde0..018e931636 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/QueryPlannerTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/planner/QueryPlannerTest.java @@ -38,6 +38,8 @@ import org.apache.logging.log4j.core.config.builder.api.LoggerComponentBuilder; import org.apache.logging.log4j.core.config.builder.api.RootLoggerComponentBuilder; import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration; +import org.apache.lucene.search.TotalHits; +import org.apache.lucene.search.TotalHits.Relation; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.search.ClearScrollRequestBuilder; import org.elasticsearch.action.search.ClearScrollResponse; @@ -162,7 +164,7 @@ public ClearScrollResponse answer(InvocationOnMock invocation) throws Throwable }); List hits = plan(sql).execute(); - return new SearchHits(hits.toArray(new SearchHit[0]), hits.size(), 0); + return new SearchHits(hits.toArray(new SearchHit[0]), new TotalHits(hits.size(), Relation.EQUAL_TO), 0); } protected QueryPlanner plan(String sql) { @@ -221,7 +223,7 @@ public SearchHits answer(InvocationOnMock invocation) { curBatch = currentBatch(); callCnt++; } - return new SearchHits(curBatch, allHits.length, 0); + return new SearchHits(curBatch, new TotalHits(allHits.length, Relation.EQUAL_TO), 0); } private boolean isNoMoreBatch() {