From 8e0b30ac39236111304e79f3728ee0b7f737cde7 Mon Sep 17 00:00:00 2001 From: penghuo Date: Thu, 14 May 2020 14:43:39 -0700 Subject: [PATCH 1/3] Opendistro Releaes 1.8.0 --- build.gradle | 43 +++++++++++++++---- gradle.properties | 2 +- gradle/wrapper/gradle-wrapper.properties | 2 +- .../sql/plugin/RestSqlAction.java | 16 ++++--- .../sql/plugin/RestSqlSettingsAction.java | 11 ++++- .../sql/plugin/RestSqlStatsAction.java | 12 +++++- .../sql/plugin/SqlPlug.java | 10 +++-- .../sql/esintgtest/QueryIT.java | 2 +- .../sql/esintgtest/TestUtils.java | 17 ++++++++ 9 files changed, 91 insertions(+), 24 deletions(-) diff --git a/build.gradle b/build.gradle index e44b767e45..af3b82c1b6 100644 --- a/build.gradle +++ b/build.gradle @@ -15,7 +15,7 @@ buildscript { ext { - es_version = "7.6.1" + es_version = "7.7.0" } // This isn't applying from repositories.gradle so repeating it here repositories { @@ -28,7 +28,7 @@ buildscript { } plugins { - id 'nebula.ospackage' version "8.1.0" + id 'nebula.ospackage' version "8.3.0" id 'java-library' id 'checkstyle' id "io.freefair.lombok" version "5.0.0-rc4" @@ -40,23 +40,50 @@ repositories { } ext { - opendistroVersion = '1.7.0' + opendistroVersion = '1.8.0' isSnapshot = "true" == System.getProperty("build.snapshot", "true") } -version = "${opendistroVersion}.0" +version = "${opendistroVersion}.0" apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.testclusters' apply plugin: 'jacoco' -if (!System.properties.containsKey('tests.rest.cluster') && !System.properties.containsKey('tests.cluster')){ - apply from: 'build-tools/sqlplugin-coverage.gradle' -} apply plugin: 'antlr' jacoco { - toolVersion = "0.8.3" + toolVersion = "0.8.5" +} +jacocoTestReport { + reports { + xml.enabled false + csv.enabled false + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, + exclude: ['**/antlr/parser/**']) + })) + } +} +test.finalizedBy(project.tasks.jacocoTestReport) +jacocoTestCoverageVerification { + violationRules { + rule { + limit { + minimum = 0.5 + } + + } + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, + exclude: ['**/antlr/parser/**']) + })) + } } +check.dependsOn jacocoTestCoverageVerification ext { projectSubstitutions = [:] diff --git a/gradle.properties b/gradle.properties index 4b54b908e9..c740b5d54c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,4 +13,4 @@ # permissions and limitations under the License. # -version=1.7.0 +version=1.8.0 diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ef1cc2387c..cc3d06cf2c 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -13,7 +13,7 @@ # permissions and limitations under the License. # -distributionUrl=https\://services.gradle.org/distributions/gradle-6.2.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.4-all.zip distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStorePath=wrapper/dists diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlAction.java index 4423a27083..51663fa3c8 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlAction.java @@ -41,6 +41,7 @@ import com.amazon.opendistroforelasticsearch.sql.utils.JsonPrettyFormatter; import com.amazon.opendistroforelasticsearch.sql.utils.LogUtils; import com.amazon.opendistroforelasticsearch.sql.utils.QueryDataAnonymizer; +import com.google.common.collect.ImmutableList; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.client.Client; @@ -58,6 +59,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -88,15 +90,19 @@ public class RestSqlAction extends BaseRestHandler { public static final String CURSOR_CLOSE_ENDPOINT = QUERY_API_ENDPOINT + "/close"; RestSqlAction(Settings settings, RestController restController) { - super(); - restController.registerHandler(RestRequest.Method.POST, QUERY_API_ENDPOINT, this); - restController.registerHandler(RestRequest.Method.POST, EXPLAIN_API_ENDPOINT, this); - restController.registerHandler(RestRequest.Method.POST, CURSOR_CLOSE_ENDPOINT, this); - this.allowExplicitIndex = MULTI_ALLOW_EXPLICIT_INDEX.get(settings); } + @Override + public List routes() { + return ImmutableList.of( + new Route(RestRequest.Method.POST, QUERY_API_ENDPOINT), + new Route(RestRequest.Method.POST, EXPLAIN_API_ENDPOINT), + new Route(RestRequest.Method.POST, CURSOR_CLOSE_ENDPOINT) + ); + } + @Override public String getName() { return "sql_action"; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlSettingsAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlSettingsAction.java index 5a977ac729..d4174e06a1 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlSettingsAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlSettingsAction.java @@ -16,8 +16,8 @@ package com.amazon.opendistroforelasticsearch.sql.plugin; import com.amazon.opendistroforelasticsearch.sql.executor.format.ErrorMessageFactory; - import com.amazon.opendistroforelasticsearch.sql.utils.LogUtils; +import com.google.common.collect.ImmutableList; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchGenerationException; @@ -39,6 +39,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; @@ -63,7 +64,6 @@ public class RestSqlSettingsAction extends BaseRestHandler { public RestSqlSettingsAction(Settings settings, RestController restController) { super(); - restController.registerHandler(RestRequest.Method.PUT, SETTINGS_API_ENDPOINT, this); } @Override @@ -71,6 +71,13 @@ public String getName() { return "sql_settings_action"; } + @Override + public List routes() { + return ImmutableList.of( + new Route(RestRequest.Method.PUT, SETTINGS_API_ENDPOINT) + ); + } + /** * @see org.elasticsearch.rest.action.admin.cluster.RestClusterUpdateSettingsAction */ diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlStatsAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlStatsAction.java index 516787f6d2..f4baab24b5 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlStatsAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlStatsAction.java @@ -18,6 +18,7 @@ import com.amazon.opendistroforelasticsearch.sql.executor.format.ErrorMessageFactory; import com.amazon.opendistroforelasticsearch.sql.metrics.Metrics; import com.amazon.opendistroforelasticsearch.sql.utils.LogUtils; +import com.google.common.collect.ImmutableList; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.client.node.NodeClient; @@ -30,6 +31,7 @@ import java.util.Arrays; import java.util.HashSet; +import java.util.List; import java.util.Set; import static org.elasticsearch.rest.RestStatus.SERVICE_UNAVAILABLE; @@ -48,8 +50,6 @@ public class RestSqlStatsAction extends BaseRestHandler { public RestSqlStatsAction(Settings settings, RestController restController) { super(); - restController.registerHandler(RestRequest.Method.POST, STATS_API_ENDPOINT, this); - restController.registerHandler(RestRequest.Method.GET, STATS_API_ENDPOINT, this); } @Override @@ -57,6 +57,14 @@ public String getName() { return "sql_stats_action"; } + @Override + public List routes() { + return ImmutableList.of( + new Route(RestRequest.Method.POST, STATS_API_ENDPOINT), + new Route(RestRequest.Method.GET, STATS_API_ENDPOINT) + ); + } + @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SqlPlug.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SqlPlug.java index a241de596f..00e657409a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SqlPlug.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SqlPlug.java @@ -85,13 +85,15 @@ public List getRestHandlers(Settings settings, RestController restC @Override public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, ResourceWatcherService resourceWatcherService, - ScriptService scriptService, NamedXContentRegistry xContentRegistry, - Environment environment, NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry) { + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver) { LocalClusterState.state().setClusterService(clusterService); LocalClusterState.state().setSqlSettings(sqlSettings); return super.createComponents(client, clusterService, threadPool, resourceWatcherService, scriptService, - xContentRegistry, environment, nodeEnvironment, namedWriteableRegistry); + xContentRegistry, environment, nodeEnvironment, namedWriteableRegistry, indexNameExpressionResolver); } @Override diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/QueryIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/QueryIT.java index 3663794320..b3d7adfd87 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/QueryIT.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/QueryIT.java @@ -1243,7 +1243,7 @@ public void queryWithAtFieldOnWhere() throws IOException { @Test public void queryWithDotAtStartOfIndexName() throws Exception { - TestUtils.createIndexByRestClient(client(), ".bank", null); + TestUtils.createHiddenIndexByRestClient(client(), ".bank", null); TestUtils.loadDataByRestClient(client(), ".bank", "/src/test/resources/.bank.json"); String response = executeQuery("SELECT education FROM .bank WHERE account_number = 12345", diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestUtils.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestUtils.java index 6be3768075..ac4825e058 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestUtils.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestUtils.java @@ -60,6 +60,23 @@ public static void createIndexByRestClient(RestClient client, String indexName, performRequest(client, request); } + /** + * https://github.com/elastic/elasticsearch/pull/49959 + * Deprecate creation of dot-prefixed index names except for hidden and system indices. + * Create hidden index by REST client. + * @param client client connection + * @param indexName test index name + * @param mapping test index mapping or null if no predefined mapping + */ + public static void createHiddenIndexByRestClient(RestClient client, String indexName, String mapping) { + Request request = new Request("PUT", "/" + indexName); + JSONObject jsonObject = isNullOrEmpty(mapping) ? new JSONObject() : new JSONObject(mapping); + jsonObject.put("settings", new JSONObject("{\"index\":{\"hidden\":true}}")); + request.setJsonEntity(jsonObject.toString()); + + performRequest(client, request); + } + /** * Check if index already exists by ES index exists API which returns: * 200 - specified indices or aliases exist From b82845f6dd04e8bfa737346e8252ce9267152c2a Mon Sep 17 00:00:00 2001 From: penghuo Date: Thu, 14 May 2020 14:54:59 -0700 Subject: [PATCH 2/3] update to JDK 14 --- .github/workflows/release-workflow.yml | 2 +- .github/workflows/test-and-build-workflow.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-workflow.yml b/.github/workflows/release-workflow.yml index 969b98b78e..9b4e4ba61f 100644 --- a/.github/workflows/release-workflow.yml +++ b/.github/workflows/release-workflow.yml @@ -9,7 +9,7 @@ jobs: build: strategy: matrix: - java: [13] + java: [14] name: Build and Release SQL Plugin runs-on: [ubuntu-16.04] diff --git a/.github/workflows/test-and-build-workflow.yml b/.github/workflows/test-and-build-workflow.yml index be045e0f95..0c825af4b1 100644 --- a/.github/workflows/test-and-build-workflow.yml +++ b/.github/workflows/test-and-build-workflow.yml @@ -10,10 +10,10 @@ jobs: steps: - uses: actions/checkout@v1 - - name: Set up JDK 1.13 + - name: Set up JDK 1.14 uses: actions/setup-java@v1 with: - java-version: 1.13 + java-version: 1.14 - name: Build with Gradle run: ./gradlew build assemble From e62cde778ae364983d3ae3353ffec032e9325d7c Mon Sep 17 00:00:00 2001 From: penghuo Date: Thu, 14 May 2020 15:03:04 -0700 Subject: [PATCH 3/3] update release notes --- opendistro-elasticsearch-sql.release-notes.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/opendistro-elasticsearch-sql.release-notes.md b/opendistro-elasticsearch-sql.release-notes.md index 5818050e99..cd314880e8 100644 --- a/opendistro-elasticsearch-sql.release-notes.md +++ b/opendistro-elasticsearch-sql.release-notes.md @@ -1,4 +1,10 @@ -## 2020-05-04 Version 1.7.0.0 (Current) +## 2020-05-14 Version 1.8.0.0 (Current) + +### Features +#### Elasticsearch Compatibility +* Feature [#467](https://github.com/opendistro-for-elasticsearch/sql/issues/467): Elasticsearch 7.7.0 compatibility + +## 2020-05-04 Version 1.7.0.0 #### Breaking Changes * Change [#414](https://github.com/opendistro-for-elasticsearch/sql/pull/414): Invalidate HTTP GET method.