From 551d8d6b2b5b3a004646e68e17a4e5b806fca755 Mon Sep 17 00:00:00 2001 From: Arsen Mkrtchyan <48698181+arsen-es@users.noreply.github.com> Date: Tue, 23 Apr 2019 11:49:00 -0700 Subject: [PATCH] Adding an integ test to check unicode characters in GET request (#38) Issue: None NLPchina/elasticsearch-sql#781 New integ test to execute sql query by calling the GET endpoint as well as passing unicode characters (this is to verify we don't have NLPchina/elasticsearch-sql#781 issue). + Minor update in loggin setup. Tested: successful build with all tests --- .../sql/executor/AsyncRestExecutor.java | 2 +- .../executor/ElasticDefaultRestExecutor.java | 2 +- .../sql/esintgtest/GetEndpointQueryIT.java | 53 +++++++++++++++++++ .../sql/esintgtest/SQLIntegTestCase.java | 51 +++++++++++++++--- src/test/resources/accounts.json | 2 +- 5 files changed, 99 insertions(+), 11 deletions(-) create mode 100644 src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/GetEndpointQueryIT.java diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/AsyncRestExecutor.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/AsyncRestExecutor.java index 6bc730c61c..c19cef58b0 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/AsyncRestExecutor.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/AsyncRestExecutor.java @@ -43,7 +43,7 @@ public class AsyncRestExecutor implements RestExecutor { /** Custom thread pool name managed by ES */ public static final String SQL_WORKER_THREAD_POOL_NAME = "sql-worker"; - private static final Logger LOG = LogManager.getLogger(); + private static final Logger LOG = LogManager.getLogger(AsyncRestExecutor.class); /** Treat all actions as blocking which means async all actions, ex. execute() in csv executor or pretty format executor */ private static final Predicate ALL_ACTION_IS_BLOCKING = anyAction -> true; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/ElasticDefaultRestExecutor.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/ElasticDefaultRestExecutor.java index e965a0384d..0d4f80f53e 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/ElasticDefaultRestExecutor.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/ElasticDefaultRestExecutor.java @@ -51,7 +51,7 @@ public class ElasticDefaultRestExecutor implements RestExecutor { /** Request builder to generate ES DSL */ private final SqlElasticRequestBuilder requestBuilder; - private static final Logger LOG = LogManager.getLogger(); + private static final Logger LOG = LogManager.getLogger(ElasticDefaultRestExecutor.class); public ElasticDefaultRestExecutor(QueryAction queryAction) { // Put explain() here to make it run in NIO thread diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/GetEndpointQueryIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/GetEndpointQueryIT.java new file mode 100644 index 0000000000..b910533ccd --- /dev/null +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/GetEndpointQueryIT.java @@ -0,0 +1,53 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.esintgtest; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.Locale; + +import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_ACCOUNT; +import static org.hamcrest.Matchers.equalTo; + +/** + * Tests to cover requests with "?format=csv" parameter + */ +public class GetEndpointQueryIT extends SQLIntegTestCase { + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + } + + @Test + public void unicodeTermInQuery() throws IOException { + + // NOTE: There are unicode characters in name, not just whitespace. + final String name = "盛虹"; + final String query = String.format(Locale.ROOT, "SELECT id, firstname FROM %s " + + "WHERE firstname=matchQuery('%s') LIMIT 2", TEST_INDEX_ACCOUNT, name); + + final JSONObject result = executeQueryWithGetRequest(query); + final JSONArray hits = getHits(result); + Assert.assertThat(hits.length(), equalTo(1)); + Assert.assertThat(hits.query("/0/_id"), equalTo("919")); + Assert.assertThat(hits.query("/0/_source/firstname"), equalTo(name)); + } +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java index 1687beb98b..262979f98d 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java @@ -34,9 +34,13 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.UnsupportedEncodingException; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.URL; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.Locale; import static com.amazon.opendistroforelasticsearch.sql.plugin.RestSqlAction.EXPLAIN_API_ENDPOINT; import static com.amazon.opendistroforelasticsearch.sql.plugin.RestSqlAction.QUERY_API_ENDPOINT; @@ -99,6 +103,23 @@ protected Request getSqlRequest(String request, boolean explain) { return sqlRequest; } + protected Request buildGetEndpointRequest(final String sqlQuery) { + + final String utf8CharsetName = StandardCharsets.UTF_8.name(); + String urlEncodedQuery = ""; + + try { + urlEncodedQuery = URLEncoder.encode(sqlQuery, utf8CharsetName); + } catch (UnsupportedEncodingException e) { + // Will never reach here since UTF-8 is always supported + Assert.fail(utf8CharsetName + " not available"); + } + + final String requestUrl = String.format(Locale.ROOT, "%s?sql=%s", + QUERY_API_ENDPOINT, urlEncodedQuery); + return new Request("GET", requestUrl); + } + protected JSONObject executeQuery(final String sqlQuery) throws IOException { final String requestBody = makeRequest(sqlQuery); @@ -129,23 +150,37 @@ protected String executeExplainRequest(final String requestBody) throws IOExcept private String executeRequest(final String requestBody, final boolean isExplainQuery) throws IOException { - RestClient restClient = ESIntegTestCase.getRestClient(); Request sqlRequest = getSqlRequest(requestBody, isExplainQuery); - Response sqlResponse = restClient.performRequest(sqlRequest); + return executeRequest(sqlRequest); + } - Assert.assertTrue(sqlResponse.getStatusLine().getStatusCode() == 200); + private String executeRequest(final Request request) throws IOException { - InputStream is = sqlResponse.getEntity().getContent(); - StringBuilder sb = new StringBuilder(); - try (BufferedReader br = new BufferedReader(new InputStreamReader(is))) { - String line = null; - while((line = br.readLine()) != null) { + RestClient restClient = ESIntegTestCase.getRestClient(); + Response response = restClient.performRequest(request); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + + final StringBuilder sb = new StringBuilder(); + + try (final InputStream is = response.getEntity().getContent(); + final BufferedReader br = new BufferedReader(new InputStreamReader(is))) { + + String line; + while ((line = br.readLine()) != null) { sb.append(line); } } + return sb.toString(); } + protected JSONObject executeQueryWithGetRequest(final String sqlQuery) throws IOException { + + final Request request = buildGetEndpointRequest(sqlQuery); + final String result = executeRequest(request); + return new JSONObject(result); + } + private String makeRequest(String query) { return String.format("{\n" + " \"query\": \"%s\"\n" + diff --git a/src/test/resources/accounts.json b/src/test/resources/accounts.json index 1331170ede..be968f0f8b 100644 --- a/src/test/resources/accounts.json +++ b/src/test/resources/accounts.json @@ -1965,7 +1965,7 @@ {"index":{"_type": "account", "_id":"914"}} {"account_number":914,"balance":7120,"firstname":"Esther","lastname":"Bean","age":32,"gender":"F","address":"583 Macon Street","employer":"Applica","email":"estherbean@applica.com","city":"Homeworth","state":"MN"} {"index":{"_type": "account", "_id":"919"}} -{"account_number":919,"balance":39655,"firstname":"Shauna","lastname":"Hanson","age":27,"gender":"M","address":"557 Hart Place","employer":"Exospace","email":"shaunahanson@exospace.com","city":"Outlook","state":"LA"} +{"account_number":919,"balance":39655,"firstname":"盛虹","lastname":"Hanson","age":27,"gender":"M","address":"557 Hart Place","employer":"Exospace","email":"shaunahanson@exospace.com","city":"Outlook","state":"LA"} {"index":{"_type": "account", "_id":"921"}} {"account_number":921,"balance":49119,"firstname":"Barbara","lastname":"Wade","age":29,"gender":"M","address":"687 Hoyts Lane","employer":"Roughies","email":"barbarawade@roughies.com","city":"Sattley","state":"CO"} {"index":{"_type": "account", "_id":"926"}}