From ffa499215deec1fe56b0260f666e2fba053f31b3 Mon Sep 17 00:00:00 2001
From: Weicong Sun <weicongs@amazon.com>
Date: Mon, 18 May 2020 16:25:55 -0700
Subject: [PATCH 1/6] Enable security IT

---
 .../esintgtest/FGACEnabledODFETestCase.java   | 120 ++++++++++++++++++
 .../sql/esintgtest/SQLIntegTestCase.java      |   4 +-
 2 files changed, 122 insertions(+), 2 deletions(-)
 create mode 100644 src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/FGACEnabledODFETestCase.java

diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/FGACEnabledODFETestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/FGACEnabledODFETestCase.java
new file mode 100644
index 0000000000..00bb7cabb3
--- /dev/null
+++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/FGACEnabledODFETestCase.java
@@ -0,0 +1,120 @@
+package com.amazon.opendistroforelasticsearch.sql.esintgtest;
+
+import org.apache.http.Header;
+import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
+import org.apache.http.ssl.SSLContexts;
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+import java.io.IOException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.util.Map;
+
+public abstract class FGACEnabledODFETestCase extends ESRestTestCase {
+    protected String getProtocol() {
+        return "https";
+    }
+
+
+    protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException {
+        RestClientBuilder builder = RestClient.builder(hosts);
+        configureCustomClient(builder, settings);
+        builder.setStrictDeprecationMode(true);
+        return builder.build();
+    }
+
+    protected static void wipeAllODFEIndices() throws IOException {
+        Response response = client().performRequest(new Request("GET", "/_cat/indices?format=json"));
+        JSONArray jsonArray = new JSONArray(EntityUtils.toString(response.getEntity(), "UTF-8"));
+        for (Object object : jsonArray) {
+            JSONObject jsonObject = (JSONObject)object;
+            String indexName = jsonObject.getString("index");
+            if (!".opendistro_security".equals(indexName)) {
+                client().performRequest(new Request("DELETE", "/" + indexName));
+            }
+        }
+    }
+    protected static void configureCustomClient(RestClientBuilder builder, Settings settings) throws IOException {
+        Map<String, String> headers = ThreadContext.buildDefaultHeaders(settings);
+        Header[] defaultHeaders = new Header[headers.size()];
+        int i = 0;
+        for (Map.Entry<String, String> entry : headers.entrySet()) {
+            defaultHeaders[i++] = new BasicHeader(entry.getKey(), entry.getValue());
+        }
+        builder.setDefaultHeaders(defaultHeaders);
+        builder.setHttpClientConfigCallback(httpClientBuilder -> {
+            SSLContext sslcontext = null;
+            try {
+                sslcontext = SSLContext.getInstance("TLS");
+                try {
+                    sslcontext.init(
+                            null, new TrustManager[]{
+                                new X509TrustManager(){
+
+                                    @Override
+                                    public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
+
+                                    }
+
+                                    @Override
+                                    public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
+
+                                    }
+
+                                    @Override
+                                    public X509Certificate[] getAcceptedIssuers() {
+                                        return new X509Certificate[0];
+                                    }
+                                }
+                            }
+                            ,
+                            null);
+                } catch (KeyManagementException e) {
+                    e.printStackTrace();
+                }
+            } catch (NoSuchAlgorithmException e) {
+                e.printStackTrace();
+            }
+            SSLIOSessionStrategy sessionStrategy = new SSLIOSessionStrategy(sslcontext);
+
+            CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+            credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("admin", "admin"));
+            return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)
+                    .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE)
+                    .setSSLStrategy(sessionStrategy);
+        });
+
+        final String socketTimeoutString = settings.get(CLIENT_SOCKET_TIMEOUT);
+        final TimeValue socketTimeout =
+                TimeValue.parseTimeValue(socketTimeoutString == null ? "60s" : socketTimeoutString, CLIENT_SOCKET_TIMEOUT);
+        builder.setRequestConfigCallback(conf -> conf.setSocketTimeout(Math.toIntExact(socketTimeout.getMillis())));
+        if (settings.hasValue(CLIENT_PATH_PREFIX)) {
+            builder.setPathPrefix(settings.get(CLIENT_PATH_PREFIX));
+        }
+    }
+}
diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java
index d395053e16..d7122917be 100644
--- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java
+++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java
@@ -78,7 +78,7 @@
  *                                \                      \
  *   XXXTIT:                  3) init()             5) init()
  */
-public abstract class SQLIntegTestCase extends ESRestTestCase {
+public abstract class SQLIntegTestCase extends FGACEnabledODFETestCase {
 
     public static final String PERSISTENT = "persistent";
     public static final String TRANSIENT = "transient";
@@ -141,7 +141,7 @@ public static void dumpCoverage() {
      */
     @AfterClass
     public static void cleanUpIndices() throws IOException {
-        wipeAllIndices();
+        wipeAllODFEIndices();
         wipeAllClusterSettings();
     }
 

From e2cd3468ac61aec6e4ac29e974156920ce752ec8 Mon Sep 17 00:00:00 2001
From: Weicong Sun <weicongs@amazon.com>
Date: Mon, 18 May 2020 17:34:16 -0700
Subject: [PATCH 2/6] Refactor the codes

---
 .../esintgtest/FGACEnabledODFETestCase.java   | 48 +++++--------------
 1 file changed, 11 insertions(+), 37 deletions(-)

diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/FGACEnabledODFETestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/FGACEnabledODFETestCase.java
index 00bb7cabb3..df8fbfc895 100644
--- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/FGACEnabledODFETestCase.java
+++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/FGACEnabledODFETestCase.java
@@ -10,6 +10,7 @@
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.message.BasicHeader;
 import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
+import org.apache.http.ssl.SSLContextBuilder;
 import org.apache.http.ssl.SSLContexts;
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;
@@ -30,6 +31,7 @@
 import javax.net.ssl.X509TrustManager;
 import java.io.IOException;
 import java.security.KeyManagementException;
+import java.security.KeyStoreException;
 import java.security.NoSuchAlgorithmException;
 import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
@@ -68,45 +70,17 @@ protected static void configureCustomClient(RestClientBuilder builder, Settings
         }
         builder.setDefaultHeaders(defaultHeaders);
         builder.setHttpClientConfigCallback(httpClientBuilder -> {
-            SSLContext sslcontext = null;
-            try {
-                sslcontext = SSLContext.getInstance("TLS");
-                try {
-                    sslcontext.init(
-                            null, new TrustManager[]{
-                                new X509TrustManager(){
-
-                                    @Override
-                                    public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
-
-                                    }
-
-                                    @Override
-                                    public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
-
-                                    }
-
-                                    @Override
-                                    public X509Certificate[] getAcceptedIssuers() {
-                                        return new X509Certificate[0];
-                                    }
-                                }
-                            }
-                            ,
-                            null);
-                } catch (KeyManagementException e) {
-                    e.printStackTrace();
-                }
-            } catch (NoSuchAlgorithmException e) {
-                e.printStackTrace();
-            }
-            SSLIOSessionStrategy sessionStrategy = new SSLIOSessionStrategy(sslcontext);
-
             CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
             credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("admin", "admin"));
-            return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)
-                    .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE)
-                    .setSSLStrategy(sessionStrategy);
+            try {
+                return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)
+                        .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE)
+                        .setSSLContext(SSLContextBuilder.create()
+                                .loadTrustMaterial(null, (chains, authType) -> true)
+                                .build());
+            } catch (Exception e) {
+               throw new RuntimeException(e);
+            }
         });
 
         final String socketTimeoutString = settings.get(CLIENT_SOCKET_TIMEOUT);

From 25649e9bfa38c8c3ed9c936cf51ab7d4dabd5ea2 Mon Sep 17 00:00:00 2001
From: Weicong Sun <weicongs@amazon.com>
Date: Tue, 19 May 2020 13:09:42 -0700
Subject: [PATCH 3/6] update

---
 ...DFETestCase.java => ODFERestTestCase.java} | 51 ++++++++++++-------
 .../sql/esintgtest/SQLIntegTestCase.java      |  3 +-
 2 files changed, 33 insertions(+), 21 deletions(-)
 rename src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/{FGACEnabledODFETestCase.java => ODFERestTestCase.java} (69%)

diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/FGACEnabledODFETestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java
similarity index 69%
rename from src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/FGACEnabledODFETestCase.java
rename to src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java
index df8fbfc895..a3980f85b1 100644
--- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/FGACEnabledODFETestCase.java
+++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java
@@ -6,18 +6,14 @@
 import org.apache.http.auth.UsernamePasswordCredentials;
 import org.apache.http.client.CredentialsProvider;
 import org.apache.http.conn.ssl.NoopHostnameVerifier;
-import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.message.BasicHeader;
-import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
 import org.apache.http.ssl.SSLContextBuilder;
-import org.apache.http.ssl.SSLContexts;
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientBuilder;
-import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -25,27 +21,37 @@
 import org.json.JSONArray;
 import org.json.JSONObject;
 
-import javax.net.ssl.KeyManager;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.X509TrustManager;
 import java.io.IOException;
-import java.security.KeyManagementException;
-import java.security.KeyStoreException;
-import java.security.NoSuchAlgorithmException;
-import java.security.cert.CertificateException;
-import java.security.cert.X509Certificate;
 import java.util.Map;
+import java.util.Optional;
 
-public abstract class FGACEnabledODFETestCase extends ESRestTestCase {
-    protected String getProtocol() {
-        return "https";
+public abstract class ODFERestTestCase extends ESRestTestCase {
+
+    protected boolean isHttps() {
+        boolean isHttps = Optional.ofNullable(System.getProperty("https"))
+                .map("true"::equalsIgnoreCase).orElse(false);
+        if (isHttps) {
+            //currently only external cluster is supported for security enabled testing
+            if (!Optional.ofNullable(System.getProperty("tests.rest.cluster")).isPresent()) {
+                throw new RuntimeException("external cluster url should be provided for security enabled testing");
+            }
+        }
+
+        return isHttps;
     }
 
+    protected String getProtocol() {
+        return isHttps() ? "https" : "http";
+    }
 
     protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException {
         RestClientBuilder builder = RestClient.builder(hosts);
-        configureCustomClient(builder, settings);
+        if (isHttps()) {
+            configureHttpsClient(builder, settings);
+        } else {
+            configureClient(builder, settings);
+        }
+
         builder.setStrictDeprecationMode(true);
         return builder.build();
     }
@@ -56,12 +62,14 @@ protected static void wipeAllODFEIndices() throws IOException {
         for (Object object : jsonArray) {
             JSONObject jsonObject = (JSONObject)object;
             String indexName = jsonObject.getString("index");
+            //.opendistro_security isn't allowed to delete from cluster
             if (!".opendistro_security".equals(indexName)) {
                 client().performRequest(new Request("DELETE", "/" + indexName));
             }
         }
     }
-    protected static void configureCustomClient(RestClientBuilder builder, Settings settings) throws IOException {
+
+    protected static void configureHttpsClient(RestClientBuilder builder, Settings settings) throws IOException {
         Map<String, String> headers = ThreadContext.buildDefaultHeaders(settings);
         Header[] defaultHeaders = new Header[headers.size()];
         int i = 0;
@@ -70,10 +78,15 @@ protected static void configureCustomClient(RestClientBuilder builder, Settings
         }
         builder.setDefaultHeaders(defaultHeaders);
         builder.setHttpClientConfigCallback(httpClientBuilder -> {
+            String userName = Optional.ofNullable(System.getProperty("user"))
+                    .orElseThrow(() -> new RuntimeException("user name is missing"));
+            String password = Optional.ofNullable(System.getProperty("password"))
+                    .orElseThrow(() -> new RuntimeException("password is missing"));
             CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
-            credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("admin", "admin"));
+            credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, password));
             try {
                 return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)
+                        //disable the certificate since our testing cluster just uses the default security configuration
                         .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE)
                         .setSSLContext(SSLContextBuilder.create()
                                 .loadTrustMaterial(null, (chains, authType) -> true)
diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java
index d7122917be..30c3e709f2 100644
--- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java
+++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
-import org.elasticsearch.test.rest.ESRestTestCase;
 import org.json.JSONArray;
 import org.json.JSONObject;
 import org.junit.AfterClass;
@@ -78,7 +77,7 @@
  *                                \                      \
  *   XXXTIT:                  3) init()             5) init()
  */
-public abstract class SQLIntegTestCase extends FGACEnabledODFETestCase {
+public abstract class SQLIntegTestCase extends ODFERestTestCase {
 
     public static final String PERSISTENT = "persistent";
     public static final String TRANSIENT = "transient";

From ffe91f934ee09a419cd1a19350eb687e12a71ee5 Mon Sep 17 00:00:00 2001
From: Weicong Sun <weicongs@amazon.com>
Date: Tue, 19 May 2020 13:31:08 -0700
Subject: [PATCH 4/6] add system properties

---
 build.gradle | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/build.gradle b/build.gradle
index e44b767e45..8626836fc3 100644
--- a/build.gradle
+++ b/build.gradle
@@ -126,6 +126,10 @@ integTest.runner {
     // allows integration test classes to access test resource from project root path
     systemProperty('project.root', project.rootDir.absolutePath)
 
+    systemProperty "https", System.getProperty("https")
+    systemProperty "user", System.getProperty("user")
+    systemProperty "password", System.getProperty("password")
+
     // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for
     // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time.
     doFirst { systemProperty 'cluster.debug', getDebug()}
@@ -151,7 +155,7 @@ integTest.runner {
         testLogging.showStandardStreams true
 
         // Pass down system properties to IT class
-        systemProperty "esHost", System.getProperty("esHost")
+            systemProperty "esHost", System.getProperty("esHost")
         systemProperty "dbUrl", System.getProperty("dbUrl")
         systemProperty "otherDbUrls", System.getProperty("otherDbUrls")
         systemProperty "queries", System.getProperty("queries")

From bde950ac5e2a0a4a180ce5e396b1526575b32af7 Mon Sep 17 00:00:00 2001
From: Weicong Sun <weicongs@amazon.com>
Date: Tue, 19 May 2020 14:43:41 -0700
Subject: [PATCH 5/6] reformat and add java doc

---
 build.gradle                                                   | 2 +-
 .../sql/esintgtest/ODFERestTestCase.java                       | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/build.gradle b/build.gradle
index 8626836fc3..50c2f5da34 100644
--- a/build.gradle
+++ b/build.gradle
@@ -155,7 +155,7 @@ integTest.runner {
         testLogging.showStandardStreams true
 
         // Pass down system properties to IT class
-            systemProperty "esHost", System.getProperty("esHost")
+        systemProperty "esHost", System.getProperty("esHost")
         systemProperty "dbUrl", System.getProperty("dbUrl")
         systemProperty "otherDbUrls", System.getProperty("otherDbUrls")
         systemProperty "queries", System.getProperty("queries")
diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java
index a3980f85b1..905e862d9b 100644
--- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java
+++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java
@@ -25,6 +25,9 @@
 import java.util.Map;
 import java.util.Optional;
 
+/**
+ * ODFE integration test base class to support both security disabled and enabled ODFE cluster.
+ */
 public abstract class ODFERestTestCase extends ESRestTestCase {
 
     protected boolean isHttps() {

From d0be9df1be147e8d3a8b9472fdcd99faa0c0722e Mon Sep 17 00:00:00 2001
From: Weicong Sun <weicongs@amazon.com>
Date: Tue, 19 May 2020 15:26:09 -0700
Subject: [PATCH 6/6] Add licence for the new file

---
 .../sql/esintgtest/ODFERestTestCase.java          | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java
index 905e862d9b..e5aa1c1b68 100644
--- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java
+++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ODFERestTestCase.java
@@ -1,3 +1,18 @@
+/*
+ *   Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License").
+ *   You may not use this file except in compliance with the License.
+ *   A copy of the License is located at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file. This file is distributed
+ *   on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *   express or implied. See the License for the specific language governing
+ *   permissions and limitations under the License.
+ */
+
 package com.amazon.opendistroforelasticsearch.sql.esintgtest;
 
 import org.apache.http.Header;