Rest mutual auth fix (#279)

* Add required mutual auth to gRPC Server/Client

Previously we had 1 sided TLS on the server side. Data between the
client and server was send over an encrypted channel, but any client
could make requests to the server.

This commit changes the behavior so that only clients with the matching
certificates can make requests to the server when TLS is enabled. This
commit does NOT add support for installing a trust manager. That must be
added in the future.

* Add full mutual auth to gRPC client/server and augment tests

* Implement mutual auth for the REST endpoints

This commit makes the PerformanceAnalyzerWebServer authenticate clients
if the user specifies a certificate authority. It also properly sets up
the server's identity, so that any clients can authenticate the server.

* Fix merge issue with WireHopperTest

* Fixing up PerformanceAnalyzerWebServerTest

* Modify gradle.yml for testing

* Remove info log flooding from testing gradle.yml

.github/workflows/gradle.yml

@@ -32,7 +32,7 @@ jobs:
         java-version: 1.12
     - name: Build RCA with Gradle
       working-directory:  ./tmp/rca
-      run: ./gradlew build
+      run: ./gradlew build --stacktrace
     - name: Generate Jacoco coverage report
       working-directory: ./tmp/rca
       run: ./gradlew jacocoTestReport

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/CertificateUtils.java

@@ -21,7 +21,12 @@
 import java.security.KeyStore;
 import java.security.PrivateKey;
 import java.security.cert.Certificate;
+import java.security.cert.X509Certificate;
 import javax.annotation.Nullable;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.X509TrustManager;
+
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.bouncycastle.asn1.pkcs.PrivateKeyInfo;
@@ -32,7 +37,7 @@
 
 public class CertificateUtils {
 
-  public static final String ALIAS_PRIVATE = "private";
+  public static final String ALIAS_IDENTITY = "identity";
   public static final String ALIAS_CERT = "cert";
   // The password is not used to encrypt keys on disk.
   public static final String IN_MEMORY_PWD = "opendistro";
@@ -65,14 +70,46 @@ public static PrivateKey getPrivateKey(final FileReader keyReader) throws Except
   public static KeyStore createKeyStore() throws Exception {
     String certFilePath = PluginSettings.instance().getSettingValue(CERTIFICATE_FILE_PATH);
     String keyFilePath = PluginSettings.instance().getSettingValue(PRIVATE_KEY_FILE_PATH);
+    KeyStore.ProtectionParameter protParam = new KeyStore.PasswordProtection(
+            CertificateUtils.IN_MEMORY_PWD.toCharArray());
     PrivateKey pk = getPrivateKey(new FileReader(keyFilePath));
     KeyStore ks = createEmptyStore();
     Certificate certificate = getCertificate(new FileReader(certFilePath));
-    ks.setCertificateEntry(ALIAS_CERT, certificate);
-    ks.setKeyEntry(ALIAS_PRIVATE, pk, IN_MEMORY_PWD.toCharArray(), new Certificate[] {certificate});
+    ks.setEntry(ALIAS_IDENTITY, new KeyStore.PrivateKeyEntry(pk, new Certificate[]{certificate}), protParam);
     return ks;
   }
 
+  public static TrustManager[] getTrustManagers(boolean forServer) throws Exception {
+    // If a certificate authority is specified, create an authenticating trust manager
+    String certificateAuthority;
+    if (forServer) {
+      certificateAuthority = PluginSettings.instance().getSettingValue(TRUSTED_CAS_FILE_PATH);
+    } else {
+      certificateAuthority = PluginSettings.instance().getSettingValue(CLIENT_TRUSTED_CAS_FILE_PATH);
+    }
+    if (certificateAuthority != null && !certificateAuthority.isEmpty()) {
+      KeyStore ks = createEmptyStore();
+      Certificate certificate = getCertificate(new FileReader(certificateAuthority));
+      ks.setCertificateEntry(ALIAS_CERT, certificate);
+      TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+      tmf.init(ks);
+      return tmf.getTrustManagers();
+    }
+    // Otherwise, return an all-trusting TrustManager
+    return new TrustManager[] {
+            new X509TrustManager() {
+
+              public X509Certificate[] getAcceptedIssuers() {
+                return null;
+              }
+
+              public void checkClientTrusted(X509Certificate[] certs, String authType) {}
+
+              public void checkServerTrusted(X509Certificate[] certs, String authType) {}
+            }
+    };
+  }
+
   public static KeyStore createEmptyStore() throws Exception {
     KeyStore ks = KeyStore.getInstance("JKS");
     ks.load(null, IN_MEMORY_PWD.toCharArray());

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/PerformanceAnalyzerWebServer.java

@@ -16,22 +16,25 @@
 package com.amazon.opendistro.elasticsearch.performanceanalyzer;
 
 import com.amazon.opendistro.elasticsearch.performanceanalyzer.config.PluginSettings;
+import com.google.common.annotations.VisibleForTesting;
 import com.sun.net.httpserver.HttpServer;
 import com.sun.net.httpserver.HttpsConfigurator;
+import com.sun.net.httpserver.HttpsParameters;
 import com.sun.net.httpserver.HttpsServer;
+
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.security.KeyStore;
 import java.security.Security;
-import java.security.cert.X509Certificate;
 import java.util.concurrent.Executors;
+
 import javax.net.ssl.HostnameVerifier;
 import javax.net.ssl.HttpsURLConnection;
 import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLSession;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.X509TrustManager;
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.SSLParameters;
+
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.bouncycastle.jce.provider.BouncyCastleProvider;
@@ -40,8 +43,10 @@ public class PerformanceAnalyzerWebServer {
 
   private static final Logger LOG = LogManager.getLogger(PerformanceAnalyzerWebServer.class);
   public static final int WEBSERVICE_DEFAULT_PORT = 9600;
-  public static final String WEBSERVICE_PORT_CONF_NAME = "webservice-listener-port";
+  @VisibleForTesting
   public static final String WEBSERVICE_BIND_HOST_NAME = "webservice-bind-host";
+  @VisibleForTesting
+  public static final String WEBSERVICE_PORT_CONF_NAME = "webservice-listener-port";
   // Use system default for max backlog.
   private static final int INCOMING_QUEUE_LENGTH = 1;
 
@@ -66,8 +71,34 @@ public static HttpServer createInternalServer(String portFromSetting, String hos
     return null;
   }
 
+  /**
+   * ClientAuthConfigurator makes the server perform client authentication if the user has set up a
+   * certificate authority
+   */
+  private static  class ClientAuthConfigurator extends HttpsConfigurator {
+    public ClientAuthConfigurator(SSLContext sslContext) {
+      super(sslContext);
+    }
+
+    @Override
+    public  void configure(HttpsParameters params) {
+      final SSLParameters sslParams = getSSLContext().getDefaultSSLParameters();
+      if (CertificateUtils.getTrustedCasFile() != null) {
+        LOG.debug("Enabling client auth");
+        final SSLEngine sslEngine = getSSLContext().createSSLEngine();
+        sslParams.setNeedClientAuth(true);
+        sslParams.setCipherSuites(sslEngine.getEnabledCipherSuites());
+        sslParams.setProtocols(sslEngine.getEnabledProtocols());
+        params.setSSLParameters(sslParams);
+      } else {
+        LOG.debug("Not enabling client auth");
+        super.configure(params);
+      }
+    }
+  }
+
   private static HttpServer createHttpsServer(int readerPort, String bindHost) throws Exception {
-    HttpsServer server = null;
+    HttpsServer server;
     if (bindHost != null && !bindHost.trim().isEmpty()) {
       LOG.info("Binding to Interface: {}", bindHost);
       server =
@@ -81,38 +112,30 @@ private static HttpServer createHttpsServer(int readerPort, String bindHost) thr
       server = HttpsServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), readerPort), INCOMING_QUEUE_LENGTH);
     }
 
-    TrustManager[] trustAllCerts =
-        new TrustManager[] {
-            new X509TrustManager() {
-
-              public X509Certificate[] getAcceptedIssuers() {
-                return null;
-              }
-
-              public void checkClientTrusted(X509Certificate[] certs, String authType) {}
-
-              public void checkServerTrusted(X509Certificate[] certs, String authType) {}
-            }
-        };
-
-    HostnameVerifier allHostsValid =
-        new HostnameVerifier() {
-          public boolean verify(String hostname, SSLSession session) {
-            return true;
-          }
-        };
-
     // Install the all-trusting trust manager
     SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
 
     KeyStore ks = CertificateUtils.createKeyStore();
     KeyManagerFactory kmf = KeyManagerFactory.getInstance("NewSunX509");
     kmf.init(ks, CertificateUtils.IN_MEMORY_PWD.toCharArray());
-    sslContext.init(kmf.getKeyManagers(), trustAllCerts, null);
+    sslContext.init(kmf.getKeyManagers(), CertificateUtils.getTrustManagers(true), null);
+    server.setHttpsConfigurator(new ClientAuthConfigurator(sslContext));
+
+
+    // TODO ask ktkrg why this is necessary
+    // Try to set HttpsURLConnection defaults, our webserver can still run even if this block fails
+    try {
+      LOG.debug("Setting default SSLSocketFactory...");
+      HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory());
+      LOG.debug("Default SSLSocketFactory set successfully");
+      HostnameVerifier allHostsValid = (hostname, session) -> true;
+      LOG.debug("Setting default HostnameVerifier...");
+      HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid);
+      LOG.debug("Default HostnameVerifier set successfully");
+    } catch (Exception e) { // Usually AccessControlException
+      LOG.warn("Exception while trying to set URLConnection defaults", e);
+    }
 
-    HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory());
-    HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid);
-    server.setHttpsConfigurator(new HttpsConfigurator(sslContext));
     return server;
   }
 

src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/config/PluginSettings.java

@@ -103,6 +103,11 @@ public boolean getHttpsEnabled() {
     return this.httpsEnabled;
   }
 
+  @VisibleForTesting
+  public void setHttpsEnabled(boolean httpsEnabled) {
+    this.httpsEnabled = httpsEnabled;
+  }
+
   @VisibleForTesting
   public void overrideProperty(String key, String value) {
     settings.setProperty(key, value);