From ffa499215deec1fe56b0260f666e2fba053f31b3 Mon Sep 17 00:00:00 2001 From: Weicong Sun <weicongs@amazon.com> Date: Mon, 18 May 2020 16:25:55 -0700 Subject: [PATCH 1/6] Enable security IT --- .../esintgtest/FGACEnabledODFETestCase.java | 120 ++++++++++++++++++ .../sql/esintgtest/SQLIntegTestCase.java | 4 +- 2 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/FGACEnabledODFETestCase.java diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/FGACEnabledODFETestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/FGACEnabledODFETestCase.java new file mode 100644 index 0000000000..00bb7cabb3 --- /dev/null +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/FGACEnabledODFETestCase.java @@ -0,0 +1,120 @@ +package com.amazon.opendistroforelasticsearch.sql.esintgtest; + +import org.apache.http.Header; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.message.BasicHeader; +import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; +import org.apache.http.ssl.SSLContexts; +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.json.JSONArray; +import org.json.JSONObject; + +import javax.net.ssl.KeyManager; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import java.io.IOException; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.Map; + +public abstract class FGACEnabledODFETestCase extends ESRestTestCase { + protected String getProtocol() { + return "https"; + } + + + protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException { + RestClientBuilder builder = RestClient.builder(hosts); + configureCustomClient(builder, settings); + builder.setStrictDeprecationMode(true); + return builder.build(); + } + + protected static void wipeAllODFEIndices() throws IOException { + Response response = client().performRequest(new Request("GET", "/_cat/indices?format=json")); + JSONArray jsonArray = new JSONArray(EntityUtils.toString(response.getEntity(), "UTF-8")); + for (Object object : jsonArray) { + JSONObject jsonObject = (JSONObject)object; + String indexName = jsonObject.getString("index"); + if (!".opendistro_security".equals(indexName)) { + client().performRequest(new Request("DELETE", "/" + indexName)); + } + } + } + protected static void configureCustomClient(RestClientBuilder builder, Settings settings) throws IOException { + Map<String, String> headers = ThreadContext.buildDefaultHeaders(settings); + Header[] defaultHeaders = new Header[headers.size()]; + int i = 0; + for (Map.Entry<String, String> entry : headers.entrySet()) { + defaultHeaders[i++] = new BasicHeader(entry.getKey(), entry.getValue()); + } + builder.setDefaultHeaders(defaultHeaders); + builder.setHttpClientConfigCallback(httpClientBuilder -> { + SSLContext sslcontext = null; + try { + sslcontext = SSLContext.getInstance("TLS"); + try { + sslcontext.init( + null, new TrustManager[]{ + new X509TrustManager(){ + + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { + + } + + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { + + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } + } + } + , + null); + } catch (KeyManagementException e) { + e.printStackTrace(); + } + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + } + SSLIOSessionStrategy sessionStrategy = new SSLIOSessionStrategy(sslcontext); + + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("admin", "admin")); + return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider) + .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .setSSLStrategy(sessionStrategy); + }); + + final String socketTimeoutString = settings.get(CLIENT_SOCKET_TIMEOUT); + final TimeValue socketTimeout = + TimeValue.parseTimeValue(socketTimeoutString == null ? "60s" : socketTimeoutString, CLIENT_SOCKET_TIMEOUT); + builder.setRequestConfigCallback(conf -> conf.setSocketTimeout(Math.toIntExact(socketTimeout.getMillis()))); + if (settings.hasValue(CLIENT_PATH_PREFIX)) { + builder.setPathPrefix(settings.get(CLIENT_PATH_PREFIX)); + } + } +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java index d395053e16..d7122917be 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java @@ -78,7 +78,7 @@ * \ \ * XXXTIT: 3) init() 5) init() */ -public abstract class SQLIntegTestCase extends ESRestTestCase { +public abstract class SQLIntegTestCase extends FGACEnabledODFETestCase { public static final String PERSISTENT = "persistent"; public static final String TRANSIENT = "transient"; @@ -141,7 +141,7 @@ public static void dumpCoverage() { */ @AfterClass public static void cleanUpIndices() throws IOException { - wipeAllIndices(); + wipeAllODFEIndices(); wipeAllClusterSettings(); } From e2cd3468ac61aec6e4ac29e974156920ce752ec8 Mon Sep 17 00:00:00 2001 From: Weicong Sun <weicongs@amazon.com> Date: Mon, 18 May 2020 17:34:16 -0700 Subject: [PATCH 2/6] Refactor the codes --- .../esintgtest/FGACEnabledODFETestCase.java | 48 +++++-------------- 1 file changed, 11 insertions(+), 37 deletions(-) diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/FGACEnabledODFETestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/FGACEnabledODFETestCase.java index 00bb7cabb3..df8fbfc895 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/FGACEnabledODFETestCase.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/FGACEnabledODFETestCase.java @@ -10,6 +10,7 @@ import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.message.BasicHeader; import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; +import org.apache.http.ssl.SSLContextBuilder; import org.apache.http.ssl.SSLContexts; import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; @@ -30,6 +31,7 @@ import javax.net.ssl.X509TrustManager; import java.io.IOException; import java.security.KeyManagementException; +import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; @@ -68,45 +70,17 @@ protected static void configureCustomClient(RestClientBuilder builder, Settings } builder.setDefaultHeaders(defaultHeaders); builder.setHttpClientConfigCallback(httpClientBuilder -> { - SSLContext sslcontext = null; - try { - sslcontext = SSLContext.getInstance("TLS"); - try { - sslcontext.init( - null, new TrustManager[]{ - new X509TrustManager(){ - - @Override - public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { - - } - - @Override - public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { - - } - - @Override - public X509Certificate[] getAcceptedIssuers() { - return new X509Certificate[0]; - } - } - } - , - null); - } catch (KeyManagementException e) { - e.printStackTrace(); - } - } catch (NoSuchAlgorithmException e) { - e.printStackTrace(); - } - SSLIOSessionStrategy sessionStrategy = new SSLIOSessionStrategy(sslcontext); - CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("admin", "admin")); - return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider) - .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) - .setSSLStrategy(sessionStrategy); + try { + return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider) + .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .setSSLContext(SSLContextBuilder.create() + .loadTrustMaterial(null, (chains, authType) -> true) + .build()); + } catch (Exception e) { + throw new RuntimeException(e); + } }); final String socketTimeoutString = settings.get(CLIENT_SOCKET_TIMEOUT); From 25649e9bfa38c8c3ed9c936cf51ab7d4dabd5ea2 Mon Sep 17 00:00:00 2001 From: Weicong Sun <weicongs@amazon.com> Date: Tue, 19 May 2020 13:09:42 -0700 Subject: [PATCH 3/6] update --- ...DFETestCase.java => ODFERestTestCase.java} | 51 ++++++++++++------- .../sql/esintgtest/SQLIntegTestCase.java | 3 +- 2 files changed, 33 insertions(+), 21 deletions(-) rename src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/{FGACEnabledODFETestCase.java => ODFERestTestCase.java} (69%) diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/FGACEnabledODFETestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java similarity index 69% rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/FGACEnabledODFETestCase.java rename to src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java index df8fbfc895..a3980f85b1 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/FGACEnabledODFETestCase.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java @@ -6,18 +6,14 @@ import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.message.BasicHeader; -import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; import org.apache.http.ssl.SSLContextBuilder; -import org.apache.http.ssl.SSLContexts; import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; -import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -25,27 +21,37 @@ import org.json.JSONArray; import org.json.JSONObject; -import javax.net.ssl.KeyManager; -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; import java.io.IOException; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; -import java.security.cert.X509Certificate; import java.util.Map; +import java.util.Optional; -public abstract class FGACEnabledODFETestCase extends ESRestTestCase { - protected String getProtocol() { - return "https"; +public abstract class ODFERestTestCase extends ESRestTestCase { + + protected boolean isHttps() { + boolean isHttps = Optional.ofNullable(System.getProperty("https")) + .map("true"::equalsIgnoreCase).orElse(false); + if (isHttps) { + //currently only external cluster is supported for security enabled testing + if (!Optional.ofNullable(System.getProperty("tests.rest.cluster")).isPresent()) { + throw new RuntimeException("external cluster url should be provided for security enabled testing"); + } + } + + return isHttps; } + protected String getProtocol() { + return isHttps() ? "https" : "http"; + } protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException { RestClientBuilder builder = RestClient.builder(hosts); - configureCustomClient(builder, settings); + if (isHttps()) { + configureHttpsClient(builder, settings); + } else { + configureClient(builder, settings); + } + builder.setStrictDeprecationMode(true); return builder.build(); } @@ -56,12 +62,14 @@ protected static void wipeAllODFEIndices() throws IOException { for (Object object : jsonArray) { JSONObject jsonObject = (JSONObject)object; String indexName = jsonObject.getString("index"); + //.opendistro_security isn't allowed to delete from cluster if (!".opendistro_security".equals(indexName)) { client().performRequest(new Request("DELETE", "/" + indexName)); } } } - protected static void configureCustomClient(RestClientBuilder builder, Settings settings) throws IOException { + + protected static void configureHttpsClient(RestClientBuilder builder, Settings settings) throws IOException { Map<String, String> headers = ThreadContext.buildDefaultHeaders(settings); Header[] defaultHeaders = new Header[headers.size()]; int i = 0; @@ -70,10 +78,15 @@ protected static void configureCustomClient(RestClientBuilder builder, Settings } builder.setDefaultHeaders(defaultHeaders); builder.setHttpClientConfigCallback(httpClientBuilder -> { + String userName = Optional.ofNullable(System.getProperty("user")) + .orElseThrow(() -> new RuntimeException("user name is missing")); + String password = Optional.ofNullable(System.getProperty("password")) + .orElseThrow(() -> new RuntimeException("password is missing")); CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("admin", "admin")); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, password)); try { return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider) + //disable the certificate since our testing cluster just uses the default security configuration .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) .setSSLContext(SSLContextBuilder.create() .loadTrustMaterial(null, (chains, authType) -> true) diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java index d7122917be..30c3e709f2 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java @@ -19,7 +19,6 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; -import org.elasticsearch.test.rest.ESRestTestCase; import org.json.JSONArray; import org.json.JSONObject; import org.junit.AfterClass; @@ -78,7 +77,7 @@ * \ \ * XXXTIT: 3) init() 5) init() */ -public abstract class SQLIntegTestCase extends FGACEnabledODFETestCase { +public abstract class SQLIntegTestCase extends ODFERestTestCase { public static final String PERSISTENT = "persistent"; public static final String TRANSIENT = "transient"; From ffe91f934ee09a419cd1a19350eb687e12a71ee5 Mon Sep 17 00:00:00 2001 From: Weicong Sun <weicongs@amazon.com> Date: Tue, 19 May 2020 13:31:08 -0700 Subject: [PATCH 4/6] add system properties --- build.gradle | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index e44b767e45..8626836fc3 100644 --- a/build.gradle +++ b/build.gradle @@ -126,6 +126,10 @@ integTest.runner { // allows integration test classes to access test resource from project root path systemProperty('project.root', project.rootDir.absolutePath) + systemProperty "https", System.getProperty("https") + systemProperty "user", System.getProperty("user") + systemProperty "password", System.getProperty("password") + // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time. doFirst { systemProperty 'cluster.debug', getDebug()} @@ -151,7 +155,7 @@ integTest.runner { testLogging.showStandardStreams true // Pass down system properties to IT class - systemProperty "esHost", System.getProperty("esHost") + systemProperty "esHost", System.getProperty("esHost") systemProperty "dbUrl", System.getProperty("dbUrl") systemProperty "otherDbUrls", System.getProperty("otherDbUrls") systemProperty "queries", System.getProperty("queries") From bde950ac5e2a0a4a180ce5e396b1526575b32af7 Mon Sep 17 00:00:00 2001 From: Weicong Sun <weicongs@amazon.com> Date: Tue, 19 May 2020 14:43:41 -0700 Subject: [PATCH 5/6] reformat and add java doc --- build.gradle | 2 +- .../sql/esintgtest/ODFERestTestCase.java | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 8626836fc3..50c2f5da34 100644 --- a/build.gradle +++ b/build.gradle @@ -155,7 +155,7 @@ integTest.runner { testLogging.showStandardStreams true // Pass down system properties to IT class - systemProperty "esHost", System.getProperty("esHost") + systemProperty "esHost", System.getProperty("esHost") systemProperty "dbUrl", System.getProperty("dbUrl") systemProperty "otherDbUrls", System.getProperty("otherDbUrls") systemProperty "queries", System.getProperty("queries") diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java index a3980f85b1..905e862d9b 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java @@ -25,6 +25,9 @@ import java.util.Map; import java.util.Optional; +/** + * ODFE integration test base class to support both security disabled and enabled ODFE cluster. + */ public abstract class ODFERestTestCase extends ESRestTestCase { protected boolean isHttps() { From d0be9df1be147e8d3a8b9472fdcd99faa0c0722e Mon Sep 17 00:00:00 2001 From: Weicong Sun <weicongs@amazon.com> Date: Tue, 19 May 2020 15:26:09 -0700 Subject: [PATCH 6/6] Add licence for the new file --- .../sql/esintgtest/ODFERestTestCase.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java index 905e862d9b..e5aa1c1b68 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java @@ -1,3 +1,18 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + package com.amazon.opendistroforelasticsearch.sql.esintgtest; import org.apache.http.Header;