Limit the scope of BouncyCastle dependency (#30959)

* Limit the scope of BouncyCastle dependency (#30358)

Limits the scope of the runtime dependency on
BouncyCastle so that it can be eventually removed.

* Splits functionality related to reading and generating certificates
and keys in two utility classes so that reading certificates and
keys doesn't require BouncyCastle.
* Implements a class for parsing PEM Encoded key material (which also
adds support for reading PKCS8 encoded encrypted private keys).
* Removes BouncyCastle dependency for all of our test suites(except
for the tests that explicitly test certificate generation) by using
pre-generated keys/certificates/keystores.

* Ensure intended key is selected in SamlAuthenticatorTests (#30993)

Uses a specific keypair for tests that require a purposefully wrong
keypair instead of selecting one randomly from the same pull from
which the correct one is selected. Entropy is low because of the
small space and the same key can be randomly selected as both the
correct one and the wrong one, causing the tests to fail.
The purposefully wrong key is also used in 
testSigningKeyIsReloadedForEachRequest and needs to be cleaned
up afterwards so the rest of the tests don't use that for signing.

x-pack/plugin/core/build.gradle

@@ -88,6 +88,7 @@ compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-
 licenseHeaders {
   approvedLicenses << 'BCrypt (BSD-like)'
   additionalLicense 'BCRYP', 'BCrypt (BSD-like)', 'Copyright (c) 2006 Damien Miller <[email protected]>'
+  excludes << 'org/elasticsearch/xpack/core/ssl/DerParser.java'
 }
 
 // make LicenseSigner available for testing signed licenses

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java

@@ -7,7 +7,6 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.SetOnce;
-import org.bouncycastle.operator.OperatorCreationException;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequest;
@@ -125,7 +124,7 @@ public Void run() {
 
     public XPackPlugin(
             final Settings settings,
-            final Path configPath) throws IOException, DestroyFailedException, OperatorCreationException, GeneralSecurityException {
+            final Path configPath) {
         super(settings);
         this.settings = settings;
         this.transportClientMode = transportClientMode(settings);

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertParsingUtils.java

@@ -0,0 +1,286 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.ssl;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.X509ExtendedKeyManager;
+import javax.net.ssl.X509ExtendedTrustManager;
+import java.io.IOException;
+import java.io.InputStream;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.Key;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.PrivateKey;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateException;
+import java.security.cert.CertificateFactory;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings.getKeyStoreType;
+
+public class CertParsingUtils {
+
+    private CertParsingUtils() {
+        throw new IllegalStateException("Utility class should not be instantiated");
+    }
+    /**
+     * Resolves a path with or without an {@link Environment} as we may be running in a transport client where we do not have access to
+     * the environment
+     */
+    @SuppressForbidden(reason = "we don't have the environment to resolve files from when running in a transport client")
+    static Path resolvePath(String path, @Nullable Environment environment) {
+        if (environment != null) {
+            return environment.configFile().resolve(path);
+        }
+        return PathUtils.get(path).normalize();
+    }
+
+    static KeyStore readKeyStore(Path path, String type, char[] password)
+            throws IOException, KeyStoreException, CertificateException, NoSuchAlgorithmException {
+        try (InputStream in = Files.newInputStream(path)) {
+            KeyStore store = KeyStore.getInstance(type);
+            assert password != null;
+            store.load(in, password);
+            return store;
+        }
+    }
+
+    /**
+     * Reads the provided paths and parses them into {@link Certificate} objects
+     *
+     * @param certPaths   the paths to the PEM encoded certificates
+     * @param environment the environment to resolve files against. May be {@code null}
+     * @return an array of {@link Certificate} objects
+     */
+    public static Certificate[] readCertificates(List<String> certPaths, @Nullable Environment environment)
+            throws CertificateException, IOException {
+        final List<Path> resolvedPaths = certPaths.stream().map(p -> resolvePath(p, environment)).collect(Collectors.toList());
+        return readCertificates(resolvedPaths);
+    }
+
+    public static Certificate[] readCertificates(List<Path> certPaths) throws CertificateException, IOException {
+        Collection<Certificate> certificates = new ArrayList<>();
+        CertificateFactory certFactory = CertificateFactory.getInstance("X.509");
+        for (Path path : certPaths) {
+            try (InputStream input = Files.newInputStream(path)) {
+                certificates.addAll((Collection<Certificate>) certFactory.generateCertificates(input));
+            }
+        }
+        return certificates.toArray(new Certificate[0]);
+    }
+
+    public static X509Certificate[] readX509Certificates(List<Path> certPaths) throws CertificateException, IOException {
+        Collection<X509Certificate> certificates = new ArrayList<>();
+        CertificateFactory certFactory = CertificateFactory.getInstance("X.509");
+        for (Path path : certPaths) {
+            try (InputStream input = Files.newInputStream(path)) {
+                certificates.addAll((Collection<X509Certificate>) certFactory.generateCertificates(input));
+            }
+        }
+        return certificates.toArray(new X509Certificate[0]);
+    }
+
+    static List<Certificate> readCertificates(InputStream input) throws CertificateException, IOException {
+        CertificateFactory certFactory = CertificateFactory.getInstance("X.509");
+        Collection<Certificate> certificates = (Collection<Certificate>) certFactory.generateCertificates(input);
+        return new ArrayList<>(certificates);
+    }
+
+    /**
+     * Read all certificate-key pairs from a PKCS#12 container.
+     *
+     * @param path        The path to the PKCS#12 container file.
+     * @param password    The password for the container file
+     * @param keyPassword A supplier for the password for each key. The key alias is supplied as an argument to the function, and it should
+     *                    return the password for that key. If it returns {@code null}, then the key-pair for that alias is not read.
+     */
+    public static Map<Certificate, Key> readPkcs12KeyPairs(Path path, char[] password, Function<String, char[]> keyPassword)
+            throws CertificateException, NoSuchAlgorithmException, KeyStoreException, IOException, UnrecoverableKeyException {
+        final KeyStore store = readKeyStore(path, "PKCS12", password);
+        final Enumeration<String> enumeration = store.aliases();
+        final Map<Certificate, Key> map = new HashMap<>(store.size());
+        while (enumeration.hasMoreElements()) {
+            final String alias = enumeration.nextElement();
+            if (store.isKeyEntry(alias)) {
+                final char[] pass = keyPassword.apply(alias);
+                map.put(store.getCertificate(alias), store.getKey(alias, pass));
+            }
+        }
+        return map;
+    }
+
+    /**
+     * Creates a {@link KeyStore} from a PEM encoded certificate and key file
+     */
+    static KeyStore getKeyStoreFromPEM(Path certificatePath, Path keyPath, char[] keyPassword)
+            throws IOException, CertificateException, KeyStoreException, NoSuchAlgorithmException {
+        final PrivateKey key = PemUtils.readPrivateKey(keyPath, () -> keyPassword);
+        final Certificate[] certificates = readCertificates(Collections.singletonList(certificatePath));
+        return getKeyStore(certificates, key, keyPassword);
+    }
+
+    /**
+     * Returns a {@link X509ExtendedKeyManager} that is built from the provided private key and certificate chain
+     */
+    public static X509ExtendedKeyManager keyManager(Certificate[] certificateChain, PrivateKey privateKey, char[] password)
+            throws NoSuchAlgorithmException, UnrecoverableKeyException, KeyStoreException, IOException, CertificateException {
+        KeyStore keyStore = getKeyStore(certificateChain, privateKey, password);
+        return keyManager(keyStore, password, KeyManagerFactory.getDefaultAlgorithm());
+    }
+
+    private static KeyStore getKeyStore(Certificate[] certificateChain, PrivateKey privateKey, char[] password)
+            throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException {
+        KeyStore keyStore = KeyStore.getInstance("jks");
+        keyStore.load(null, null);
+        // password must be non-null for keystore...
+        keyStore.setKeyEntry("key", privateKey, password, certificateChain);
+        return keyStore;
+    }
+
+    /**
+     * Returns a {@link X509ExtendedKeyManager} that is built from the provided keystore
+     */
+    static X509ExtendedKeyManager keyManager(KeyStore keyStore, char[] password, String algorithm)
+            throws NoSuchAlgorithmException, UnrecoverableKeyException, KeyStoreException {
+        KeyManagerFactory kmf = KeyManagerFactory.getInstance(algorithm);
+        kmf.init(keyStore, password);
+        KeyManager[] keyManagers = kmf.getKeyManagers();
+        for (KeyManager keyManager : keyManagers) {
+            if (keyManager instanceof X509ExtendedKeyManager) {
+                return (X509ExtendedKeyManager) keyManager;
+            }
+        }
+        throw new IllegalStateException("failed to find a X509ExtendedKeyManager");
+    }
+
+    public static X509ExtendedKeyManager getKeyManager(X509KeyPairSettings keyPair, Settings settings,
+                                                       @Nullable String trustStoreAlgorithm, Environment environment) {
+        if (trustStoreAlgorithm == null) {
+            trustStoreAlgorithm = TrustManagerFactory.getDefaultAlgorithm();
+        }
+        final KeyConfig keyConfig = createKeyConfig(keyPair, settings, trustStoreAlgorithm);
+        if (keyConfig == null) {
+            return null;
+        } else {
+            return keyConfig.createKeyManager(environment);
+        }
+    }
+
+    static KeyConfig createKeyConfig(X509KeyPairSettings keyPair, Settings settings, String trustStoreAlgorithm) {
+        String keyPath = keyPair.keyPath.get(settings).orElse(null);
+        String keyStorePath = keyPair.keystorePath.get(settings).orElse(null);
+
+        if (keyPath != null && keyStorePath != null) {
+            throw new IllegalArgumentException("you cannot specify a keystore and key file");
+        }
+
+        if (keyPath != null) {
+            SecureString keyPassword = keyPair.keyPassword.get(settings);
+            String certPath = keyPair.certificatePath.get(settings).orElse(null);
+            if (certPath == null) {
+                throw new IllegalArgumentException("you must specify the certificates [" + keyPair.certificatePath.getKey()
+                        + "] to use with the key [" + keyPair.keyPath.getKey() + "]");
+            }
+            return new PEMKeyConfig(keyPath, keyPassword, certPath);
+        }
+
+        if (keyStorePath != null) {
+            SecureString keyStorePassword = keyPair.keystorePassword.get(settings);
+            String keyStoreAlgorithm = keyPair.keystoreAlgorithm.get(settings);
+            String keyStoreType = getKeyStoreType(keyPair.keystoreType, settings, keyStorePath);
+            SecureString keyStoreKeyPassword = keyPair.keystoreKeyPassword.get(settings);
+            if (keyStoreKeyPassword.length() == 0) {
+                keyStoreKeyPassword = keyStorePassword;
+            }
+            return new StoreKeyConfig(keyStorePath, keyStoreType, keyStorePassword, keyStoreKeyPassword, keyStoreAlgorithm,
+                    trustStoreAlgorithm);
+        }
+        return null;
+
+    }
+
+    /**
+     * Creates a {@link X509ExtendedTrustManager} based on the provided certificates
+     *
+     * @param certificates the certificates to trust
+     * @return a trust manager that trusts the provided certificates
+     */
+    public static X509ExtendedTrustManager trustManager(Certificate[] certificates)
+            throws NoSuchAlgorithmException, KeyStoreException, IOException, CertificateException {
+        KeyStore store = trustStore(certificates);
+        return trustManager(store, TrustManagerFactory.getDefaultAlgorithm());
+    }
+
+    static KeyStore trustStore(Certificate[] certificates)
+            throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException {
+        assert certificates != null : "Cannot create trust store with null certificates";
+        KeyStore store = KeyStore.getInstance("jks");
+        store.load(null, null);
+        int counter = 0;
+        for (Certificate certificate : certificates) {
+            store.setCertificateEntry("cert" + counter, certificate);
+            counter++;
+        }
+        return store;
+    }
+
+    /**
+     * Loads the truststore and creates a {@link X509ExtendedTrustManager}
+     *
+     * @param trustStorePath      the path to the truststore
+     * @param trustStorePassword  the password to the truststore
+     * @param trustStoreAlgorithm the algorithm to use for the truststore
+     * @param env                 the environment to use for file resolution. May be {@code null}
+     * @return a trust manager with the trust material from the store
+     */
+    public static X509ExtendedTrustManager trustManager(String trustStorePath, String trustStoreType, char[] trustStorePassword,
+                                                        String trustStoreAlgorithm, @Nullable Environment env)
+            throws NoSuchAlgorithmException, KeyStoreException, IOException, CertificateException {
+        KeyStore trustStore = readKeyStore(resolvePath(trustStorePath, env), trustStoreType, trustStorePassword);
+        return trustManager(trustStore, trustStoreAlgorithm);
+    }
+
+    /**
+     * Creates a {@link X509ExtendedTrustManager} based on the trust material in the provided {@link KeyStore}
+     */
+    static X509ExtendedTrustManager trustManager(KeyStore keyStore, String algorithm)
+            throws NoSuchAlgorithmException, KeyStoreException {
+        TrustManagerFactory tmf = TrustManagerFactory.getInstance(algorithm);
+        tmf.init(keyStore);
+        TrustManager[] trustManagers = tmf.getTrustManagers();
+        for (TrustManager trustManager : trustManagers) {
+            if (trustManager instanceof X509ExtendedTrustManager) {
+                return (X509ExtendedTrustManager) trustManager;
+            }
+        }
+        throw new IllegalStateException("failed to find a X509ExtendedTrustManager");
+    }
+}