From 2aa1204f99f1eb9c2c66a6249365dc4ec74b7168 Mon Sep 17 00:00:00 2001 From: Andor Molnar Date: Thu, 1 Sep 2022 11:48:38 +0200 Subject: [PATCH] HBASE-27346. Add PEM reader, test and spotless fixes --- .../hbase/io/crypto/tls/BCFKSFileLoader.java | 27 ++- .../io/crypto/tls/FileKeyStoreLoader.java | 90 ++++---- .../FileKeyStoreLoaderBuilderProvider.java | 48 ++-- .../hbase/io/crypto/tls/JKSFileLoader.java | 26 +-- .../hbase/io/crypto/tls/KeyStoreLoader.java | 46 ++-- .../hbase/io/crypto/tls/PEMFileLoader.java | 52 ++--- .../hbase/io/crypto/tls/PKCS12FileLoader.java | 27 ++- .../hadoop/hbase/io/crypto/tls/PemReader.java | 214 ++++++++++++++++++ .../tls/StandardTypeFileKeyStoreLoader.java | 73 +++--- .../hadoop/hbase/io/crypto/tls/X509Util.java | 29 +-- .../hbase/io/crypto/tls/TestX509Util.java | 115 ++++++++-- .../hbase/io/crypto/tls/X509TestHelpers.java | 2 +- 12 files changed, 507 insertions(+), 242 deletions(-) create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/PemReader.java diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/BCFKSFileLoader.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/BCFKSFileLoader.java index b8c5338a26b1..a231848d9cd3 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/BCFKSFileLoader.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/BCFKSFileLoader.java @@ -15,7 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.hbase.io.crypto.tls; /** @@ -23,21 +22,21 @@ *

* This file has been copied from the Apache ZooKeeper project. * @see Base - * revision + * "https://github.com/apache/zookeeper/blob/c74658d398cdc1d207aa296cb6e20de00faec03e/zookeeper-server/src/main/java/org/apache/zookeeper/common/BCFKSFileLoader.java">Base + * revision */ class BCFKSFileLoader extends StandardTypeFileKeyStoreLoader { - private BCFKSFileLoader(String keyStorePath, - String trustStorePath, - char[] keyStorePassword, - char[] trustStorePassword) { - super(keyStorePath, trustStorePath, keyStorePassword, trustStorePassword, SupportedStandardKeyFormat.BCFKS); - } + private BCFKSFileLoader(String keyStorePath, String trustStorePath, char[] keyStorePassword, + char[] trustStorePassword) { + super(keyStorePath, trustStorePath, keyStorePassword, trustStorePassword, + SupportedStandardKeyFormat.BCFKS); + } - static class Builder extends FileKeyStoreLoader.Builder { - @Override - BCFKSFileLoader build() { - return new BCFKSFileLoader(keyStorePath, trustStorePath, keyStorePassword, trustStorePassword); - } + static class Builder extends FileKeyStoreLoader.Builder { + @Override + BCFKSFileLoader build() { + return new BCFKSFileLoader(keyStorePath, trustStorePath, keyStorePassword, + trustStorePassword); } + } } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/FileKeyStoreLoader.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/FileKeyStoreLoader.java index f953e3bdf629..3a1740b4faf0 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/FileKeyStoreLoader.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/FileKeyStoreLoader.java @@ -15,68 +15,66 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.hbase.io.crypto.tls; import java.util.Objects; /** - * Base class for instances of {@link KeyStoreLoader} which load the key/trust - * stores from files on a filesystem. + * Base class for instances of {@link KeyStoreLoader} which load the key/trust stores from files on + * a filesystem. *

* This file has been copied from the Apache ZooKeeper project. * @see Base - * revision + * "https://github.com/apache/zookeeper/blob/c74658d398cdc1d207aa296cb6e20de00faec03e/zookeeper-server/src/main/java/org/apache/zookeeper/common/FileKeyStoreLoader.java">Base + * revision */ abstract class FileKeyStoreLoader implements KeyStoreLoader { - final String keyStorePath; - final String trustStorePath; - final char[] keyStorePassword; - final char[] trustStorePassword; - - FileKeyStoreLoader(String keyStorePath, - String trustStorePath, - char[] keyStorePassword, - char[] trustStorePassword) { - this.keyStorePath = keyStorePath; - this.trustStorePath = trustStorePath; - this.keyStorePassword = keyStorePassword; - this.trustStorePassword = trustStorePassword; - } + final String keyStorePath; + final String trustStorePath; + final char[] keyStorePassword; + final char[] trustStorePassword; - /** - * Base class for builder pattern used by subclasses. - * @param the subtype of FileKeyStoreLoader created by the Builder. - */ - static abstract class Builder { - String keyStorePath; - String trustStorePath; - char[] keyStorePassword; - char[] trustStorePassword; + FileKeyStoreLoader(String keyStorePath, String trustStorePath, char[] keyStorePassword, + char[] trustStorePassword) { + this.keyStorePath = keyStorePath; + this.trustStorePath = trustStorePath; + this.keyStorePassword = keyStorePassword; + this.trustStorePassword = trustStorePassword; + } - Builder() {} + /** + * Base class for builder pattern used by subclasses. + * @param the subtype of FileKeyStoreLoader created by the Builder. + */ + static abstract class Builder { + String keyStorePath; + String trustStorePath; + char[] keyStorePassword; + char[] trustStorePassword; - Builder setKeyStorePath(String keyStorePath) { - this.keyStorePath = Objects.requireNonNull(keyStorePath); - return this; - } + Builder() { + } - Builder setTrustStorePath(String trustStorePath) { - this.trustStorePath = Objects.requireNonNull(trustStorePath); - return this; - } + Builder setKeyStorePath(String keyStorePath) { + this.keyStorePath = Objects.requireNonNull(keyStorePath); + return this; + } - Builder setKeyStorePassword(char[] keyStorePassword) { - this.keyStorePassword = Objects.requireNonNull(keyStorePassword); - return this; - } + Builder setTrustStorePath(String trustStorePath) { + this.trustStorePath = Objects.requireNonNull(trustStorePath); + return this; + } - Builder setTrustStorePassword(char[] trustStorePassword) { - this.trustStorePassword = Objects.requireNonNull(trustStorePassword); - return this; - } + Builder setKeyStorePassword(char[] keyStorePassword) { + this.keyStorePassword = Objects.requireNonNull(keyStorePassword); + return this; + } - abstract T build(); + Builder setTrustStorePassword(char[] trustStorePassword) { + this.trustStorePassword = Objects.requireNonNull(trustStorePassword); + return this; } + + abstract T build(); + } } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/FileKeyStoreLoaderBuilderProvider.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/FileKeyStoreLoaderBuilderProvider.java index dcf8e56db28d..5ab672659f3e 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/FileKeyStoreLoaderBuilderProvider.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/FileKeyStoreLoaderBuilderProvider.java @@ -15,40 +15,38 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.hbase.io.crypto.tls; import java.util.Objects; +import org.apache.yetus.audience.InterfaceAudience; /** * This file has been copied from the Apache ZooKeeper project. * @see Base - * revision + * "https://github.com/apache/zookeeper/blob/c74658d398cdc1d207aa296cb6e20de00faec03e/zookeeper-server/src/main/java/org/apache/zookeeper/common/FileKeyStoreLoaderBuilderProvider.java">Base + * revision */ +@InterfaceAudience.Private public class FileKeyStoreLoaderBuilderProvider { - /** - * Returns a {@link FileKeyStoreLoader.Builder} that can build a loader - * which loads keys and certs from files of the given - * {@link KeyStoreFileType}. - * - * @param type the file type to load keys/certs from. - * @return a new Builder. - */ - static FileKeyStoreLoader.Builder + /** + * Returns a {@link FileKeyStoreLoader.Builder} that can build a loader which loads keys and certs + * from files of the given {@link KeyStoreFileType}. + * @param type the file type to load keys/certs from. + * @return a new Builder. + */ + static FileKeyStoreLoader.Builder getBuilderForKeyStoreFileType(KeyStoreFileType type) { - switch (Objects.requireNonNull(type)) { - case JKS: - return new JKSFileLoader.Builder(); - case PEM: - return new PEMFileLoader.Builder(); - case PKCS12: - return new PKCS12FileLoader.Builder(); - case BCFKS: - return new BCFKSFileLoader.Builder(); - default: - throw new AssertionError( - "Unexpected StoreFileType: " + type.name()); - } + switch (Objects.requireNonNull(type)) { + case JKS: + return new JKSFileLoader.Builder(); + case PEM: + return new PEMFileLoader.Builder(); + case PKCS12: + return new PKCS12FileLoader.Builder(); + case BCFKS: + return new BCFKSFileLoader.Builder(); + default: + throw new AssertionError("Unexpected StoreFileType: " + type.name()); } + } } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/JKSFileLoader.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/JKSFileLoader.java index 3d2339defa5c..7f52d939a9cd 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/JKSFileLoader.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/JKSFileLoader.java @@ -15,7 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.hbase.io.crypto.tls; /** @@ -23,21 +22,20 @@ *

* This file has been copied from the Apache ZooKeeper project. * @see Base - * revision + * "https://github.com/apache/zookeeper/blob/c74658d398cdc1d207aa296cb6e20de00faec03e/zookeeper-server/src/main/java/org/apache/zookeeper/common/JKSFileLoader.java">Base + * revision */ class JKSFileLoader extends StandardTypeFileKeyStoreLoader { - private JKSFileLoader(String keyStorePath, - String trustStorePath, - char[] keyStorePassword, - char[] trustStorePassword) { - super(keyStorePath, trustStorePath, keyStorePassword, trustStorePassword, SupportedStandardKeyFormat.JKS); - } + private JKSFileLoader(String keyStorePath, String trustStorePath, char[] keyStorePassword, + char[] trustStorePassword) { + super(keyStorePath, trustStorePath, keyStorePassword, trustStorePassword, + SupportedStandardKeyFormat.JKS); + } - static class Builder extends FileKeyStoreLoader.Builder { - @Override - JKSFileLoader build() { - return new JKSFileLoader(keyStorePath, trustStorePath, keyStorePassword, trustStorePassword); - } + static class Builder extends FileKeyStoreLoader.Builder { + @Override + JKSFileLoader build() { + return new JKSFileLoader(keyStorePath, trustStorePath, keyStorePassword, trustStorePassword); } + } } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/KeyStoreLoader.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/KeyStoreLoader.java index f0c3a8356438..928b3b9d0469 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/KeyStoreLoader.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/KeyStoreLoader.java @@ -15,7 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.hbase.io.crypto.tls; import java.io.IOException; @@ -27,31 +26,28 @@ *

* This file has been copied from the Apache ZooKeeper project. * @see Base - * revision + * "https://github.com/apache/zookeeper/blob/c74658d398cdc1d207aa296cb6e20de00faec03e/zookeeper-server/src/main/java/org/apache/zookeeper/common/KeyStoreLoader.java">Base + * revision */ interface KeyStoreLoader { - /** - * Loads a KeyStore which contains at least one private key and the - * associated X509 cert chain. - * - * @return a new KeyStore - * @throws IOException if loading the key store fails due to an IO error, - * such as "file not found". - * @throws GeneralSecurityException if loading the key store fails due to - * a security error, such as "unsupported crypto algorithm". - */ - KeyStore loadKeyStore() throws IOException, GeneralSecurityException; + /** + * Loads a KeyStore which contains at least one private key and the associated X509 cert chain. + * @return a new KeyStore + * @throws IOException if loading the key store fails due to an IO error, such as + * "file not found". + * @throws GeneralSecurityException if loading the key store fails due to a security error, such + * as "unsupported crypto algorithm". + */ + KeyStore loadKeyStore() throws IOException, GeneralSecurityException; - /** - * Loads a KeyStore which contains at least one X509 cert chain for a - * trusted Certificate Authority (CA). - * - * @return a new KeyStore - * @throws IOException if loading the trust store fails due to an IO error, - * such as "file not found". - * @throws GeneralSecurityException if loading the trust store fails due to - * a security error, such as "unsupported crypto algorithm". - */ - KeyStore loadTrustStore() throws IOException, GeneralSecurityException; + /** + * Loads a KeyStore which contains at least one X509 cert chain for a trusted Certificate + * Authority (CA). + * @return a new KeyStore + * @throws IOException if loading the trust store fails due to an IO error, such as + * "file not found". + * @throws GeneralSecurityException if loading the trust store fails due to a security error, such + * as "unsupported crypto algorithm". + */ + KeyStore loadTrustStore() throws IOException, GeneralSecurityException; } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/PEMFileLoader.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/PEMFileLoader.java index 479da7ef03d3..4fa8334190af 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/PEMFileLoader.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/PEMFileLoader.java @@ -15,54 +15,42 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.hbase.io.crypto.tls; import java.io.File; import java.io.IOException; import java.security.GeneralSecurityException; import java.security.KeyStore; -import java.util.Optional; - -import org.apache.zookeeper.util.PemReader; /** * Implementation of {@link FileKeyStoreLoader} that loads from PEM files. *

* This file has been copied from the Apache ZooKeeper project. * @see Base - * revision + * "https://github.com/apache/zookeeper/blob/c74658d398cdc1d207aa296cb6e20de00faec03e/zookeeper-server/src/main/java/org/apache/zookeeper/common/PEMFileLoader.java">Base + * revision */ class PEMFileLoader extends FileKeyStoreLoader { - private PEMFileLoader(String keyStorePath, - String trustStorePath, - char[] keyStorePassword, - char[] trustStorePassword) { - super(keyStorePath, trustStorePath, keyStorePassword, trustStorePassword); - } + private PEMFileLoader(String keyStorePath, String trustStorePath, char[] keyStorePassword, + char[] trustStorePassword) { + super(keyStorePath, trustStorePath, keyStorePassword, trustStorePassword); + } - @Override - public KeyStore loadKeyStore() throws IOException, GeneralSecurityException { - Optional passwordOption; - if (keyStorePassword == null || keyStorePassword.length == 0) { - passwordOption = Optional.empty(); - } else { - passwordOption = Optional.of(String.valueOf(keyStorePassword)); - } - File file = new File(keyStorePath); - return PemReader.loadKeyStore(file, file, passwordOption); - } + @Override + public KeyStore loadKeyStore() throws IOException, GeneralSecurityException { + File file = new File(keyStorePath); + return PemReader.loadKeyStore(file, file, keyStorePassword); + } - @Override - public KeyStore loadTrustStore() throws IOException, GeneralSecurityException { - return PemReader.loadTrustStore(new File(trustStorePath)); - } + @Override + public KeyStore loadTrustStore() throws IOException, GeneralSecurityException { + return PemReader.loadTrustStore(new File(trustStorePath)); + } - static class Builder extends FileKeyStoreLoader.Builder { - @Override - PEMFileLoader build() { - return new PEMFileLoader(keyStorePath, trustStorePath, keyStorePassword, trustStorePassword); - } + static class Builder extends FileKeyStoreLoader.Builder { + @Override + PEMFileLoader build() { + return new PEMFileLoader(keyStorePath, trustStorePath, keyStorePassword, trustStorePassword); } + } } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/PKCS12FileLoader.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/PKCS12FileLoader.java index 0d826fad1957..b11c948e5877 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/PKCS12FileLoader.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/PKCS12FileLoader.java @@ -15,7 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.hbase.io.crypto.tls; /** @@ -23,21 +22,21 @@ *

* This file has been copied from the Apache ZooKeeper project. * @see Base - * revision + * "https://github.com/apache/zookeeper/blob/c74658d398cdc1d207aa296cb6e20de00faec03e/zookeeper-server/src/main/java/org/apache/zookeeper/common/PKCS12FileLoader.java">Base + * revision */ class PKCS12FileLoader extends StandardTypeFileKeyStoreLoader { - private PKCS12FileLoader(String keyStorePath, - String trustStorePath, - char[] keyStorePassword, - char[] trustStorePassword) { - super(keyStorePath, trustStorePath, keyStorePassword, trustStorePassword, SupportedStandardKeyFormat.PKCS12); - } + private PKCS12FileLoader(String keyStorePath, String trustStorePath, char[] keyStorePassword, + char[] trustStorePassword) { + super(keyStorePath, trustStorePath, keyStorePassword, trustStorePassword, + SupportedStandardKeyFormat.PKCS12); + } - static class Builder extends FileKeyStoreLoader.Builder { - @Override - PKCS12FileLoader build() { - return new PKCS12FileLoader(keyStorePath, trustStorePath, keyStorePassword, trustStorePassword); - } + static class Builder extends FileKeyStoreLoader.Builder { + @Override + PKCS12FileLoader build() { + return new PKCS12FileLoader(keyStorePath, trustStorePath, keyStorePassword, + trustStorePassword); } + } } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/PemReader.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/PemReader.java new file mode 100644 index 000000000000..5d40e5c8b437 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/PemReader.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.crypto.tls; + +import static java.nio.charset.StandardCharsets.US_ASCII; +import static java.util.Base64.getMimeDecoder; +import static java.util.regex.Pattern.CASE_INSENSITIVE; +import static javax.crypto.Cipher.DECRYPT_MODE; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.security.GeneralSecurityException; +import java.security.KeyFactory; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.PrivateKey; +import java.security.PublicKey; +import java.security.cert.Certificate; +import java.security.cert.CertificateException; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.PKCS8EncodedKeySpec; +import java.security.spec.X509EncodedKeySpec; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.crypto.Cipher; +import javax.crypto.EncryptedPrivateKeyInfo; +import javax.crypto.SecretKey; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.PBEKeySpec; +import javax.security.auth.x500.X500Principal; + +/** + * This file has been copied from the Apache ZooKeeper project. + * @see Base + * revision + */ +final class PemReader { + private static final Pattern CERT_PATTERN = + Pattern.compile("-+BEGIN\\s+.*CERTIFICATE[^-]*-+(?:\\s|\\r|\\n)+" + // Header + "([a-z0-9+/=\\r\\n]+)" + // Base64 text + "-+END\\s+.*CERTIFICATE[^-]*-+", // Footer + CASE_INSENSITIVE); + + private static final Pattern PRIVATE_KEY_PATTERN = + Pattern.compile("-+BEGIN\\s+.*PRIVATE\\s+KEY[^-]*-+(?:\\s|\\r|\\n)+" + // Header + "([a-z0-9+/=\\r\\n]+)" + // Base64 text + "-+END\\s+.*PRIVATE\\s+KEY[^-]*-+", // Footer + CASE_INSENSITIVE); + + private static final Pattern PUBLIC_KEY_PATTERN = + Pattern.compile("-+BEGIN\\s+.*PUBLIC\\s+KEY[^-]*-+(?:\\s|\\r|\\n)+" + // Header + "([a-z0-9+/=\\r\\n]+)" + // Base64 text + "-+END\\s+.*PUBLIC\\s+KEY[^-]*-+", // Footer + CASE_INSENSITIVE); + + private PemReader() { + } + + public static KeyStore loadTrustStore(File certificateChainFile) + throws IOException, GeneralSecurityException { + KeyStore keyStore = KeyStore.getInstance("JKS"); + keyStore.load(null, null); + + List certificateChain = readCertificateChain(certificateChainFile); + for (X509Certificate certificate : certificateChain) { + X500Principal principal = certificate.getSubjectX500Principal(); + keyStore.setCertificateEntry(principal.getName("RFC2253"), certificate); + } + return keyStore; + } + + public static KeyStore loadKeyStore(File certificateChainFile, File privateKeyFile, + char[] keyPassword) throws IOException, GeneralSecurityException { + PrivateKey key = loadPrivateKey(privateKeyFile, keyPassword); + + List certificateChain = readCertificateChain(certificateChainFile); + if (certificateChain.isEmpty()) { + throw new CertificateException( + "Certificate file does not contain any certificates: " + certificateChainFile); + } + + KeyStore keyStore = KeyStore.getInstance("JKS"); + keyStore.load(null, null); + keyStore.setKeyEntry("key", key, keyPassword, certificateChain.toArray(new Certificate[0])); + return keyStore; + } + + public static List readCertificateChain(File certificateChainFile) + throws IOException, GeneralSecurityException { + String contents = new String(Files.readAllBytes(certificateChainFile.toPath()), US_ASCII); + return readCertificateChain(contents); + } + + public static List readCertificateChain(String certificateChain) + throws CertificateException { + Matcher matcher = CERT_PATTERN.matcher(certificateChain); + CertificateFactory certificateFactory = CertificateFactory.getInstance("X.509"); + List certificates = new ArrayList<>(); + + int start = 0; + while (matcher.find(start)) { + byte[] buffer = base64Decode(matcher.group(1)); + certificates.add( + (X509Certificate) certificateFactory.generateCertificate(new ByteArrayInputStream(buffer))); + start = matcher.end(); + } + + return certificates; + } + + public static PrivateKey loadPrivateKey(File privateKeyFile, char[] keyPassword) + throws IOException, GeneralSecurityException { + String privateKey = new String(Files.readAllBytes(privateKeyFile.toPath()), US_ASCII); + return loadPrivateKey(privateKey, keyPassword); + } + + public static PrivateKey loadPrivateKey(String privateKey, char[] keyPassword) + throws IOException, GeneralSecurityException { + Matcher matcher = PRIVATE_KEY_PATTERN.matcher(privateKey); + if (!matcher.find()) { + throw new KeyStoreException("did not find a private key"); + } + byte[] encodedKey = base64Decode(matcher.group(1)); + + PKCS8EncodedKeySpec encodedKeySpec; + if (keyPassword != null && keyPassword.length > 0) { + EncryptedPrivateKeyInfo encryptedPrivateKeyInfo = new EncryptedPrivateKeyInfo(encodedKey); + SecretKeyFactory keyFactory = + SecretKeyFactory.getInstance(encryptedPrivateKeyInfo.getAlgName()); + SecretKey secretKey = keyFactory.generateSecret(new PBEKeySpec(keyPassword)); + + Cipher cipher = Cipher.getInstance(encryptedPrivateKeyInfo.getAlgName()); + cipher.init(DECRYPT_MODE, secretKey, encryptedPrivateKeyInfo.getAlgParameters()); + + encodedKeySpec = encryptedPrivateKeyInfo.getKeySpec(cipher); + } else { + encodedKeySpec = new PKCS8EncodedKeySpec(encodedKey); + } + + // this code requires a key in PKCS8 format which is not the default openssl format + // to convert to the PKCS8 format you use : openssl pkcs8 -topk8 ... + try { + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + return keyFactory.generatePrivate(encodedKeySpec); + } catch (InvalidKeySpecException ignore) { + } + + try { + KeyFactory keyFactory = KeyFactory.getInstance("EC"); + return keyFactory.generatePrivate(encodedKeySpec); + } catch (InvalidKeySpecException ignore) { + } + + KeyFactory keyFactory = KeyFactory.getInstance("DSA"); + return keyFactory.generatePrivate(encodedKeySpec); + } + + public static PublicKey loadPublicKey(File publicKeyFile) + throws IOException, GeneralSecurityException { + String publicKey = new String(Files.readAllBytes(publicKeyFile.toPath()), US_ASCII); + return loadPublicKey(publicKey); + } + + public static PublicKey loadPublicKey(String publicKey) throws GeneralSecurityException { + Matcher matcher = PUBLIC_KEY_PATTERN.matcher(publicKey); + if (!matcher.find()) { + throw new KeyStoreException("did not find a public key"); + } + String data = matcher.group(1); + byte[] encodedKey = base64Decode(data); + + X509EncodedKeySpec encodedKeySpec = new X509EncodedKeySpec(encodedKey); + try { + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + return keyFactory.generatePublic(encodedKeySpec); + } catch (InvalidKeySpecException ignore) { + } + + try { + KeyFactory keyFactory = KeyFactory.getInstance("EC"); + return keyFactory.generatePublic(encodedKeySpec); + } catch (InvalidKeySpecException ignore) { + } + + KeyFactory keyFactory = KeyFactory.getInstance("DSA"); + return keyFactory.generatePublic(encodedKeySpec); + } + + private static byte[] base64Decode(String base64) { + return getMimeDecoder().decode(base64.getBytes(US_ASCII)); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/StandardTypeFileKeyStoreLoader.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/StandardTypeFileKeyStoreLoader.java index 63dbc811506a..67aebdd6c7bc 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/StandardTypeFileKeyStoreLoader.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/StandardTypeFileKeyStoreLoader.java @@ -15,11 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.hbase.io.crypto.tls; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; @@ -28,53 +26,54 @@ import java.security.KeyStoreException; /** - * Base class for instances of {@link KeyStoreLoader} which load the key/trust - * stores from files on a filesystem using standard {@link KeyStore} types like - * JKS or PKCS12. + * Base class for instances of {@link KeyStoreLoader} which load the key/trust stores from files on + * a filesystem using standard {@link KeyStore} types like JKS or PKCS12. *

* This file has been copied from the Apache ZooKeeper project. * @see Base - * revision + * "https://github.com/apache/zookeeper/blob/c74658d398cdc1d207aa296cb6e20de00faec03e/zookeeper-server/src/main/java/org/apache/zookeeper/common/StandardTypeFileKeyStoreLoader.java">Base + * revision */ abstract class StandardTypeFileKeyStoreLoader extends FileKeyStoreLoader { - private static final char[] EMPTY_CHAR_ARRAY = new char[0]; + private static final char[] EMPTY_CHAR_ARRAY = new char[0]; - protected final SupportedStandardKeyFormat format; + protected final SupportedStandardKeyFormat format; - protected enum SupportedStandardKeyFormat { - JKS, PKCS12, BCFKS - } + protected enum SupportedStandardKeyFormat { + JKS, + PKCS12, + BCFKS + } - StandardTypeFileKeyStoreLoader(String keyStorePath, String trustStorePath, char[] keyStorePassword, - char[] trustStorePassword, SupportedStandardKeyFormat format) { - super(keyStorePath, trustStorePath, keyStorePassword, trustStorePassword); - this.format = format; - } + StandardTypeFileKeyStoreLoader(String keyStorePath, String trustStorePath, + char[] keyStorePassword, char[] trustStorePassword, SupportedStandardKeyFormat format) { + super(keyStorePath, trustStorePath, keyStorePassword, trustStorePassword); + this.format = format; + } - @Override - public KeyStore loadKeyStore() throws IOException, GeneralSecurityException { - try (InputStream inputStream = Files.newInputStream(new File(keyStorePath).toPath())) { - KeyStore ks = keyStoreInstance(); - ks.load(inputStream, passwordStringToCharArray(keyStorePassword)); - return ks; - } + @Override + public KeyStore loadKeyStore() throws IOException, GeneralSecurityException { + try (InputStream inputStream = Files.newInputStream(new File(keyStorePath).toPath())) { + KeyStore ks = keyStoreInstance(); + ks.load(inputStream, passwordStringToCharArray(keyStorePassword)); + return ks; } + } - @Override - public KeyStore loadTrustStore() throws IOException, GeneralSecurityException { - try (InputStream inputStream = Files.newInputStream(new File(trustStorePath).toPath())) { - KeyStore ts = keyStoreInstance(); - ts.load(inputStream, passwordStringToCharArray(trustStorePassword)); - return ts; - } + @Override + public KeyStore loadTrustStore() throws IOException, GeneralSecurityException { + try (InputStream inputStream = Files.newInputStream(new File(trustStorePath).toPath())) { + KeyStore ts = keyStoreInstance(); + ts.load(inputStream, passwordStringToCharArray(trustStorePassword)); + return ts; } + } - private KeyStore keyStoreInstance() throws KeyStoreException { - return KeyStore.getInstance(format.name()); - } + private KeyStore keyStoreInstance() throws KeyStoreException { + return KeyStore.getInstance(format.name()); + } - private static char[] passwordStringToCharArray(char[] password) { - return password == null ? EMPTY_CHAR_ARRAY : password; - } + private static char[] passwordStringToCharArray(char[] password) { + return password == null ? EMPTY_CHAR_ARRAY : password; + } } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/X509Util.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/X509Util.java index 76b7fad4c596..471ad41d06fb 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/X509Util.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/tls/X509Util.java @@ -17,10 +17,7 @@ */ package org.apache.hadoop.hbase.io.crypto.tls; -import java.io.File; import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; import java.security.GeneralSecurityException; import java.security.KeyStore; import java.security.Security; @@ -227,19 +224,16 @@ public static SslContext createSslContextForServer(Configuration config) static X509KeyManager createKeyManager(String keyStoreLocation, char[] keyStorePassword, String keyStoreType) throws KeyManagerException { - if (keyStoreType == null) { - keyStoreType = "jks"; - } - if (keyStorePassword == null) { keyStorePassword = EMPTY_CHAR_ARRAY; } try { - KeyStore ks = KeyStore.getInstance(keyStoreType); - try (InputStream inputStream = Files.newInputStream(new File(keyStoreLocation).toPath())) { - ks.load(inputStream, keyStorePassword); - } + KeyStoreFileType storeFileType = + KeyStoreFileType.fromPropertyValueOrFileName(keyStoreType, keyStoreLocation); + KeyStore ks = FileKeyStoreLoaderBuilderProvider.getBuilderForKeyStoreFileType(storeFileType) + .setKeyStorePath(keyStoreLocation).setKeyStorePassword(keyStorePassword).build() + .loadKeyStore(); KeyManagerFactory kmf = KeyManagerFactory.getInstance("PKIX"); kmf.init(ks, keyStorePassword); @@ -272,19 +266,16 @@ static X509KeyManager createKeyManager(String keyStoreLocation, char[] keyStoreP static X509TrustManager createTrustManager(String trustStoreLocation, char[] trustStorePassword, String trustStoreType, boolean crlEnabled, boolean ocspEnabled) throws TrustManagerException { - if (trustStoreType == null) { - trustStoreType = "jks"; - } - if (trustStorePassword == null) { trustStorePassword = EMPTY_CHAR_ARRAY; } try { - KeyStore ts = KeyStore.getInstance(trustStoreType); - try (InputStream inputStream = Files.newInputStream(new File(trustStoreLocation).toPath())) { - ts.load(inputStream, trustStorePassword); - } + KeyStoreFileType storeFileType = + KeyStoreFileType.fromPropertyValueOrFileName(trustStoreType, trustStoreLocation); + KeyStore ts = FileKeyStoreLoaderBuilderProvider.getBuilderForKeyStoreFileType(storeFileType) + .setTrustStorePath(trustStoreLocation).setTrustStorePassword(trustStorePassword).build() + .loadTrustStore(); PKIXBuilderParameters pbParams = new PKIXBuilderParameters(ts, new X509CertSelector()); if (crlEnabled || ocspEnabled) { diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/tls/TestX509Util.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/tls/TestX509Util.java index 61134390e8a2..bb70c02946fb 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/tls/TestX509Util.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/tls/TestX509Util.java @@ -36,6 +36,8 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import javax.net.ssl.X509KeyManager; +import javax.net.ssl.X509TrustManager; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; @@ -204,6 +206,69 @@ public void testCRLDisabled() throws Exception { assertFalse(Boolean.valueOf(Security.getProperty("ocsp.enable"))); } + @Test + public void testLoadPEMKeyStore() throws Exception { + // Make sure we can instantiate a key manager from the PEM file on disk + X509KeyManager km = X509Util.createKeyManager( + x509TestContext.getKeyStoreFile(KeyStoreFileType.PEM).getAbsolutePath(), + x509TestContext.getKeyStorePassword(), KeyStoreFileType.PEM.getPropertyValue()); + } + + @Test + public void testLoadPEMKeyStoreNullPassword() throws Exception { + assumeThat(x509TestContext.getKeyStorePassword(), equalTo(EMPTY_CHAR_ARRAY)); + // Make sure that empty password and null password are treated the same + X509KeyManager km = X509Util.createKeyManager( + x509TestContext.getKeyStoreFile(KeyStoreFileType.PEM).getAbsolutePath(), null, + KeyStoreFileType.PEM.getPropertyValue()); + } + + @Test + public void testLoadPEMKeyStoreAutodetectStoreFileType() throws Exception { + // Make sure we can instantiate a key manager from the PEM file on disk + X509KeyManager km = X509Util.createKeyManager( + x509TestContext.getKeyStoreFile(KeyStoreFileType.PEM).getAbsolutePath(), + x509TestContext.getKeyStorePassword(), + null /* null StoreFileType means 'autodetect from file extension' */); + } + + @Test(expected = KeyManagerException.class) + public void testLoadPEMKeyStoreWithWrongPassword() throws Exception { + // Attempting to load with the wrong key password should fail + X509KeyManager km = X509Util.createKeyManager( + x509TestContext.getKeyStoreFile(KeyStoreFileType.PEM).getAbsolutePath(), + "wrong password".toCharArray(), // intentionally use the wrong password + KeyStoreFileType.PEM.getPropertyValue()); + } + + @Test + public void testLoadPEMTrustStore() throws Exception { + // Make sure we can instantiate a trust manager from the PEM file on disk + X509TrustManager tm = X509Util.createTrustManager( + x509TestContext.getTrustStoreFile(KeyStoreFileType.PEM).getAbsolutePath(), + x509TestContext.getTrustStorePassword(), KeyStoreFileType.PEM.getPropertyValue(), false, + false); + } + + @Test + public void testLoadPEMTrustStoreNullPassword() throws Exception { + assumeThat(x509TestContext.getTrustStorePassword(), equalTo(EMPTY_CHAR_ARRAY)); + // Make sure that empty password and null password are treated the same + X509TrustManager tm = X509Util.createTrustManager( + x509TestContext.getTrustStoreFile(KeyStoreFileType.PEM).getAbsolutePath(), null, + KeyStoreFileType.PEM.getPropertyValue(), false, false); + } + + @Test + public void testLoadPEMTrustStoreAutodetectStoreFileType() throws Exception { + // Make sure we can instantiate a trust manager from the PEM file on disk + X509TrustManager tm = X509Util.createTrustManager( + x509TestContext.getTrustStoreFile(KeyStoreFileType.PEM).getAbsolutePath(), + x509TestContext.getTrustStorePassword(), null, // null StoreFileType means 'autodetect from + // file extension' + false, false); + } + @Test public void testLoadJKSKeyStore() throws Exception { // Make sure we can instantiate a key manager from the JKS file on disk @@ -222,9 +287,9 @@ public void testLoadJKSKeyStoreNullPassword() throws Exception { } @Test - public void testLoadJKSKeyStoreFileTypeDefaultToJks() throws Exception { + public void testLoadJKSKeyStoreAutodetectStoreFileType() throws Exception { // Make sure we can instantiate a key manager from the JKS file on disk - X509Util.createKeyManager( + X509KeyManager km = X509Util.createKeyManager( x509TestContext.getKeyStoreFile(KeyStoreFileType.JKS).getAbsolutePath(), x509TestContext.getKeyStorePassword(), null /* null StoreFileType means 'autodetect from file extension' */); @@ -258,16 +323,17 @@ public void testLoadJKSTrustStoreNullPassword() throws Exception { } @Test - public void testLoadJKSTrustStoreFileTypeDefaultToJks() throws Exception { + public void testLoadJKSTrustStoreAutodetectStoreFileType() throws Exception { // Make sure we can instantiate a trust manager from the JKS file on disk - X509Util.createTrustManager( + X509TrustManager tm = X509Util.createTrustManager( x509TestContext.getTrustStoreFile(KeyStoreFileType.JKS).getAbsolutePath(), - // null StoreFileType means 'autodetect from file extension' - x509TestContext.getTrustStorePassword(), null, true, true); + x509TestContext.getTrustStorePassword(), null, // null StoreFileType means 'autodetect from + // file extension' + true, true); } @Test - public void testLoadJKSTrustStoreWithWrongPassword() throws Exception { + public void testLoadJKSTrustStoreWithWrongPassword() { assertThrows(TrustManagerException.class, () -> { // Attempting to load with the wrong key password should fail X509Util.createTrustManager( @@ -294,7 +360,16 @@ public void testLoadPKCS12KeyStoreNullPassword() throws Exception { } @Test - public void testLoadPKCS12KeyStoreWithWrongPassword() throws Exception { + public void testLoadPKCS12KeyStoreAutodetectStoreFileType() throws Exception { + // Make sure we can instantiate a key manager from the PKCS12 file on disk + X509KeyManager km = X509Util.createKeyManager( + x509TestContext.getKeyStoreFile(KeyStoreFileType.PKCS12).getAbsolutePath(), + x509TestContext.getKeyStorePassword(), + null /* null StoreFileType means 'autodetect from file extension' */); + } + + @Test + public void testLoadPKCS12KeyStoreWithWrongPassword() { assertThrows(KeyManagerException.class, () -> { // Attempting to load with the wrong key password should fail X509Util.createKeyManager( @@ -322,7 +397,17 @@ public void testLoadPKCS12TrustStoreNullPassword() throws Exception { } @Test - public void testLoadPKCS12TrustStoreWithWrongPassword() throws Exception { + public void testLoadPKCS12TrustStoreAutodetectStoreFileType() throws Exception { + // Make sure we can instantiate a trust manager from the PKCS12 file on disk + X509TrustManager tm = X509Util.createTrustManager( + x509TestContext.getTrustStoreFile(KeyStoreFileType.PKCS12).getAbsolutePath(), + x509TestContext.getTrustStorePassword(), null, // null StoreFileType means 'autodetect from + // file extension' + true, true); + } + + @Test + public void testLoadPKCS12TrustStoreWithWrongPassword() { assertThrows(TrustManagerException.class, () -> { // Attempting to load with the wrong key password should fail X509Util.createTrustManager( @@ -332,42 +417,42 @@ public void testLoadPKCS12TrustStoreWithWrongPassword() throws Exception { } @Test - public void testGetDefaultCipherSuitesJava8() throws Exception { + public void testGetDefaultCipherSuitesJava8() { String[] cipherSuites = X509Util.getDefaultCipherSuitesForJavaVersion("1.8"); // Java 8 default should have the CBC suites first assertThat(cipherSuites[0], containsString("CBC")); } @Test - public void testGetDefaultCipherSuitesJava9() throws Exception { + public void testGetDefaultCipherSuitesJava9() { String[] cipherSuites = X509Util.getDefaultCipherSuitesForJavaVersion("9"); // Java 9+ default should have the GCM suites first assertThat(cipherSuites[0], containsString("GCM")); } @Test - public void testGetDefaultCipherSuitesJava10() throws Exception { + public void testGetDefaultCipherSuitesJava10() { String[] cipherSuites = X509Util.getDefaultCipherSuitesForJavaVersion("10"); // Java 9+ default should have the GCM suites first assertThat(cipherSuites[0], containsString("GCM")); } @Test - public void testGetDefaultCipherSuitesJava11() throws Exception { + public void testGetDefaultCipherSuitesJava11() { String[] cipherSuites = X509Util.getDefaultCipherSuitesForJavaVersion("11"); // Java 9+ default should have the GCM suites first assertThat(cipherSuites[0], containsString("GCM")); } @Test - public void testGetDefaultCipherSuitesUnknownVersion() throws Exception { + public void testGetDefaultCipherSuitesUnknownVersion() { String[] cipherSuites = X509Util.getDefaultCipherSuitesForJavaVersion("notaversion"); // If version can't be parsed, use the more conservative Java 8 default assertThat(cipherSuites[0], containsString("CBC")); } @Test - public void testGetDefaultCipherSuitesNullVersion() throws Exception { + public void testGetDefaultCipherSuitesNullVersion() { assertThrows(NullPointerException.class, () -> { X509Util.getDefaultCipherSuitesForJavaVersion(null); }); diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/tls/X509TestHelpers.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/tls/X509TestHelpers.java index 1697dca8669b..42c798abcd9e 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/tls/X509TestHelpers.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/tls/X509TestHelpers.java @@ -281,7 +281,7 @@ public static String pemEncodePrivateKey(PrivateKey key, char[] password) StringWriter stringWriter = new StringWriter(); JcaPEMWriter pemWriter = new JcaPEMWriter(stringWriter); OutputEncryptor encryptor = null; - if (password != null) { + if (password != null && password.length > 0) { encryptor = new JceOpenSSLPKCS8EncryptorBuilder(PKCSObjectIdentifiers.pbeWithSHAAnd3_KeyTripleDES_CBC) .setProvider(BouncyCastleProvider.PROVIDER_NAME).setRandom(PRNG).setPasssword(password)