From ef2f8aacad5b374a92e85c890787a3056e7e8851 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 10 Oct 2019 03:03:10 +0300 Subject: [PATCH 01/29] Encrypted blob store repository - take I (#46170) POC with GCM from BC --- .../repositories/azure/AzureRepository.java | 2 +- .../gcs/GoogleCloudStorageRepository.java | 2 +- .../repositories/hdfs/HdfsRepository.java | 2 +- .../repositories/s3/S3Repository.java | 2 +- .../blobstore/BlobStoreRepository.java | 9 +- .../repositories/fs/FsRepository.java | 2 +- .../elasticsearch/xpack/core/XPackPlugin.java | 3 +- .../plugin/repository-encrypted/build.gradle | 16 + .../licenses/bc-fips-1.0.1.jar.sha1 | 1 + .../licenses/bc-fips-LICENSE.txt | 12 + .../licenses/bc-fips-NOTICE.txt | 0 .../licenses/bcpkix-fips-1.0.3.jar.sha1 | 1 + .../licenses/bcpkix-fips-LICENSE.txt | 12 + .../licenses/bcpkix-fips-NOTICE.txt | 0 .../encrypted/EncryptedRepository.java | 308 ++++++++++++++++++ .../encrypted/EncryptedRepositoryPlugin.java | 45 +++ .../plugin-metadata/plugin-security.policy | 9 + .../encrypted/EncryptedRepositoryTests.java | 14 + .../test/repository_encrypted/10_basic.yml | 16 + 19 files changed, 448 insertions(+), 8 deletions(-) create mode 100644 x-pack/plugin/repository-encrypted/build.gradle create mode 100644 x-pack/plugin/repository-encrypted/licenses/bc-fips-1.0.1.jar.sha1 create mode 100644 x-pack/plugin/repository-encrypted/licenses/bc-fips-LICENSE.txt create mode 100644 x-pack/plugin/repository-encrypted/licenses/bc-fips-NOTICE.txt create mode 100644 x-pack/plugin/repository-encrypted/licenses/bcpkix-fips-1.0.3.jar.sha1 create mode 100644 x-pack/plugin/repository-encrypted/licenses/bcpkix-fips-LICENSE.txt create mode 100644 x-pack/plugin/repository-encrypted/licenses/bcpkix-fips-NOTICE.txt create mode 100644 x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java create mode 100644 x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepositoryPlugin.java create mode 100644 x-pack/plugin/repository-encrypted/src/main/plugin-metadata/plugin-security.policy create mode 100644 x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/EncryptedRepositoryTests.java create mode 100644 x-pack/plugin/repository-encrypted/src/test/resources/rest-api-spec/test/repository_encrypted/10_basic.yml diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java index b5c6ed70ad0d2..3bbe5db68353b 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java @@ -124,7 +124,7 @@ protected AzureBlobStore createBlobStore() { } @Override - protected ByteSizeValue chunkSize() { + public ByteSizeValue chunkSize() { return chunkSize; } diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java index 4b17fd6bef3ea..e5b33b8d5fdf6 100644 --- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java +++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java @@ -94,7 +94,7 @@ protected GoogleCloudStorageBlobStore createBlobStore() { } @Override - protected ByteSizeValue chunkSize() { + public ByteSizeValue chunkSize() { return chunkSize; } diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java index 72430bcd36631..dcc3076f03893 100644 --- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java @@ -233,7 +233,7 @@ protected HdfsBlobStore createBlobStore() { } @Override - protected ByteSizeValue chunkSize() { + public ByteSizeValue chunkSize() { return chunkSize; } } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index af895758723f5..688878b6b630a 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -207,7 +207,7 @@ protected BlobStore getBlobStore() { } @Override - protected ByteSizeValue chunkSize() { + public ByteSizeValue chunkSize() { return chunkSize; } } diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 59c6a248ca0f4..162932477a4d7 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -180,6 +180,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp protected final ChecksumBlobStoreFormat snapshotFormat; + private final NamedXContentRegistry namedXContentRegistry; + private final boolean readOnly; private final ChecksumBlobStoreFormat indexShardSnapshotFormat; @@ -194,6 +196,10 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp private final BlobPath basePath; + protected BlobStoreRepository(BlobStoreRepository other) { + this(other.metadata, other.namedXContentRegistry, other.threadPool, other.basePath); + } + /** * Constructs new BlobStoreRepository * @param metadata The metadata for this repository including name and settings @@ -211,6 +217,7 @@ protected BlobStoreRepository( restoreRateLimiter = getRateLimiter(metadata.settings(), "max_restore_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB)); readOnly = metadata.settings().getAsBoolean("readonly", false); this.basePath = basePath; + this.namedXContentRegistry = namedXContentRegistry; indexShardSnapshotFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_CODEC, SNAPSHOT_NAME_FORMAT, BlobStoreIndexShardSnapshot::fromXContent, namedXContentRegistry, compress); @@ -343,7 +350,7 @@ protected final boolean isCompress() { * * @return chunk size */ - protected ByteSizeValue chunkSize() { + public ByteSizeValue chunkSize() { return null; } diff --git a/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java b/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java index 61558e4f42efa..c177de9afc6f1 100644 --- a/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java @@ -109,7 +109,7 @@ protected BlobStore createBlobStore() throws Exception { } @Override - protected ByteSizeValue chunkSize() { + public ByteSizeValue chunkSize() { return chunkSize; } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java index 1b20ceae9233e..099e06c205301 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java @@ -76,7 +76,6 @@ import java.time.Clock; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -313,7 +312,7 @@ public static Path resolveConfigFile(Environment env, String name) { @Override public Map getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry, ThreadPool threadPool) { - return Collections.singletonMap("source", SourceOnlySnapshotRepository.newRepositoryFactory()); + return Map.of("source", SourceOnlySnapshotRepository.newRepositoryFactory()); } @Override diff --git a/x-pack/plugin/repository-encrypted/build.gradle b/x-pack/plugin/repository-encrypted/build.gradle new file mode 100644 index 0000000000000..7e0014987d934 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/build.gradle @@ -0,0 +1,16 @@ +evaluationDependsOn(xpackModule('core')) + +apply plugin: 'elasticsearch.esplugin' +esplugin { + name 'repository-encrypted' + description 'Elasticsearch Expanded Pack Plugin - client-side encrypted repositories.' + classname 'org.elasticsearch.repositories.encrypted.EncryptedRepositoryPlugin' + extendedPlugins = ['x-pack-core'] +} + +dependencies { + compile "org.bouncycastle:bc-fips:1.0.1" + compile "org.bouncycastle:bcpkix-fips:1.0.3" +} + +integTest.enabled = false diff --git a/x-pack/plugin/repository-encrypted/licenses/bc-fips-1.0.1.jar.sha1 b/x-pack/plugin/repository-encrypted/licenses/bc-fips-1.0.1.jar.sha1 new file mode 100644 index 0000000000000..2e4bb227b43bc --- /dev/null +++ b/x-pack/plugin/repository-encrypted/licenses/bc-fips-1.0.1.jar.sha1 @@ -0,0 +1 @@ +ed8dd3144761eaa33b9c56f5e2bef85f1b731d6f \ No newline at end of file diff --git a/x-pack/plugin/repository-encrypted/licenses/bc-fips-LICENSE.txt b/x-pack/plugin/repository-encrypted/licenses/bc-fips-LICENSE.txt new file mode 100644 index 0000000000000..e94fe212ff725 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/licenses/bc-fips-LICENSE.txt @@ -0,0 +1,12 @@ +Copyright (c) 2000 - 2019 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/x-pack/plugin/repository-encrypted/licenses/bc-fips-NOTICE.txt b/x-pack/plugin/repository-encrypted/licenses/bc-fips-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/x-pack/plugin/repository-encrypted/licenses/bcpkix-fips-1.0.3.jar.sha1 b/x-pack/plugin/repository-encrypted/licenses/bcpkix-fips-1.0.3.jar.sha1 new file mode 100644 index 0000000000000..3262bdf0f3d03 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/licenses/bcpkix-fips-1.0.3.jar.sha1 @@ -0,0 +1 @@ +33c47b105777c9dcc8a08188186bd35401366bd1 \ No newline at end of file diff --git a/x-pack/plugin/repository-encrypted/licenses/bcpkix-fips-LICENSE.txt b/x-pack/plugin/repository-encrypted/licenses/bcpkix-fips-LICENSE.txt new file mode 100644 index 0000000000000..e94fe212ff725 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/licenses/bcpkix-fips-LICENSE.txt @@ -0,0 +1,12 @@ +Copyright (c) 2000 - 2019 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/x-pack/plugin/repository-encrypted/licenses/bcpkix-fips-NOTICE.txt b/x-pack/plugin/repository-encrypted/licenses/bcpkix-fips-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java new file mode 100644 index 0000000000000..bb1317d73260d --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java @@ -0,0 +1,308 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.repositories.encrypted; + +import org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider; +import org.elasticsearch.cluster.metadata.RepositoryMetaData; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.blobstore.BlobContainer; +import org.elasticsearch.common.blobstore.BlobMetaData; +import org.elasticsearch.common.blobstore.BlobPath; +import org.elasticsearch.common.blobstore.BlobStore; +import org.elasticsearch.common.blobstore.DeleteResult; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.hash.MessageDigests; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.settings.SecureSetting; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.blobstore.BlobStoreRepository; + +import javax.crypto.Cipher; +import javax.crypto.CipherInputStream; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.KeyGenerator; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.SecretKeySpec; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.spec.InvalidKeySpecException; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; + +public class EncryptedRepository extends BlobStoreRepository { + + static final Setting.AffixSetting ENCRYPTION_PASSWORD_SETTING = Setting.affixKeySetting("repository.encrypted.", + "password", key -> SecureSetting.secureString(key, null)); + + private static final Setting DELEGATE_TYPE = new Setting<>("delegate_type", "", Function.identity()); + private static final int GCM_TAG_BYTES_LENGTH = 16; + private static final String ENCRYPTION_MODE = "AES/GCM/NoPadding"; + private static final String ENCRYPTION_METADATA_PREFIX = "encryption-metadata-"; + // always the same IV because the key is randomly generated anew (Key-IV pair is never repeated) + //private static final GCMParameterSpec gcmParameterSpec = new GCMParameterSpec(128, new byte[] {0,1,2,3,4,5,6,7,8,9,10,11 }); + private static final IvParameterSpec ivParameterSpec = new IvParameterSpec(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }); + // given the mode, the IV and the tag length, the maximum "chunk" size is ~64GB, we set it to 32GB to err on the safe side + public static final ByteSizeValue MAX_CHUNK_SIZE = new ByteSizeValue(32, ByteSizeUnit.GB); + + private static final BouncyCastleFipsProvider BC_FIPS_PROV = new BouncyCastleFipsProvider(); + + private final BlobStoreRepository delegatedRepository; + private final SecretKey masterSecretKey; + + protected EncryptedRepository(BlobStoreRepository delegatedRepository, SecretKey masterSecretKey) { + super(delegatedRepository); + this.delegatedRepository = delegatedRepository; + this.masterSecretKey = masterSecretKey; + } + + @Override + protected BlobStore createBlobStore() throws Exception { + return new EncryptedBlobStoreDecorator(this.delegatedRepository.blobStore(), this.masterSecretKey); + } + + @Override + protected void doStart() { + this.delegatedRepository.start(); + super.doStart(); + } + + @Override + protected void doStop() { + super.doStop(); + this.delegatedRepository.stop(); + } + + @Override + protected void doClose() { + super.doClose(); + this.delegatedRepository.close(); + } + + @Override + public ByteSizeValue chunkSize() { + ByteSizeValue delegatedChunkSize = this.delegatedRepository.chunkSize(); + if (delegatedChunkSize == null || delegatedChunkSize.compareTo(MAX_CHUNK_SIZE) > 0) { + return MAX_CHUNK_SIZE; + } else { + return delegatedChunkSize; + } + } + + /** + * Returns a new encrypted repository factory + */ + public static Repository.Factory newRepositoryFactory(final Settings settings) { + final Map cachedRepositoryPasswords = new HashMap<>(); + for (String repositoryName : ENCRYPTION_PASSWORD_SETTING.getNamespaces(settings)) { + Setting encryptionPasswordSetting = ENCRYPTION_PASSWORD_SETTING + .getConcreteSettingForNamespace(repositoryName); + SecureString encryptionPassword = encryptionPasswordSetting.get(settings); + cachedRepositoryPasswords.put(repositoryName, encryptionPassword.getChars()); + } + return new Repository.Factory() { + + @Override + public Repository create(RepositoryMetaData metadata) { + throw new UnsupportedOperationException(); + } + + @Override + public Repository create(RepositoryMetaData metaData, Function typeLookup) throws Exception { + String delegateType = DELEGATE_TYPE.get(metaData.settings()); + if (Strings.hasLength(delegateType) == false) { + throw new IllegalArgumentException(DELEGATE_TYPE.getKey() + " must be set"); + } + + if (false == cachedRepositoryPasswords.containsKey(metaData.name())) { + throw new IllegalArgumentException( + ENCRYPTION_PASSWORD_SETTING.getConcreteSettingForNamespace(metaData.name()).getKey() + " must be set"); + } + SecretKey secretKey = generateSecretKeyFromPassword(cachedRepositoryPasswords.get(metaData.name())); + Repository.Factory factory = typeLookup.apply(delegateType); + Repository delegatedRepository = factory.create(new RepositoryMetaData(metaData.name(), + delegateType, metaData.settings())); + if (false == (delegatedRepository instanceof BlobStoreRepository)) { + throw new IllegalArgumentException("Unsupported type " + DELEGATE_TYPE.getKey()); + } + return new EncryptedRepository((BlobStoreRepository)delegatedRepository, secretKey); + } + }; + } + + private static class EncryptedBlobStoreDecorator implements BlobStore { + + private final BlobStore delegatedBlobStore; + private final SecretKey masterSecretKey; + + EncryptedBlobStoreDecorator(BlobStore blobStore, SecretKey masterSecretKey) { + this.delegatedBlobStore = blobStore; + this.masterSecretKey = masterSecretKey; + } + + @Override + public void close() throws IOException { + this.delegatedBlobStore.close(); + } + + @Override + public BlobContainer blobContainer(BlobPath path) { + BlobPath encryptionMetadataBlobPath = BlobPath.cleanPath(); + encryptionMetadataBlobPath = encryptionMetadataBlobPath.add(ENCRYPTION_METADATA_PREFIX + keyId(this.masterSecretKey)); + for (String pathComponent : path) { + encryptionMetadataBlobPath = encryptionMetadataBlobPath.add(pathComponent); + } + return new EncryptedBlobContainerDecorator(this.delegatedBlobStore.blobContainer(path), + this.delegatedBlobStore.blobContainer(encryptionMetadataBlobPath), this.masterSecretKey); + } + } + + private static class EncryptedBlobContainerDecorator implements BlobContainer { + + private final BlobContainer delegatedBlobContainer; + private final BlobContainer encryptionMetadataBlobContainer; + private final SecretKey masterSecretKey; + + EncryptedBlobContainerDecorator(BlobContainer delegatedBlobContainer, BlobContainer encryptionMetadataBlobContainer, + SecretKey masterSecretKey) { + this.delegatedBlobContainer = delegatedBlobContainer; + this.encryptionMetadataBlobContainer = encryptionMetadataBlobContainer; + this.masterSecretKey = masterSecretKey; + } + + @Override + public BlobPath path() { + return this.delegatedBlobContainer.path(); + } + + @Override + public InputStream readBlob(String blobName) throws IOException { + final BytesReference dataDecryptionKeyBytes = Streams.readFully(this.encryptionMetadataBlobContainer.readBlob(blobName)); + try { + SecretKey dataDecryptionKey = unwrapKey(BytesReference.toBytes(dataDecryptionKeyBytes), this.masterSecretKey); + Cipher cipher = Cipher.getInstance(ENCRYPTION_MODE, BC_FIPS_PROV); + cipher.init(Cipher.DECRYPT_MODE, dataDecryptionKey, ivParameterSpec); + return new CipherInputStream(this.delegatedBlobContainer.readBlob(blobName), cipher); + } catch (InvalidKeyException | NoSuchAlgorithmException | NoSuchPaddingException | InvalidAlgorithmParameterException e) { + throw new IOException(e); + } + } + + @Override + public void writeBlob(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) throws IOException { + try { + SecretKey dataEncryptionKey = generateRandomSecretKey(); + byte[] wrappedDataEncryptionKey = wrapKey(dataEncryptionKey, this.masterSecretKey); + try (InputStream stream = new ByteArrayInputStream(wrappedDataEncryptionKey)) { + this.encryptionMetadataBlobContainer.writeBlob(blobName, stream, wrappedDataEncryptionKey.length, failIfAlreadyExists); + } + Cipher cipher = Cipher.getInstance(ENCRYPTION_MODE, BC_FIPS_PROV); + cipher.init(Cipher.ENCRYPT_MODE, dataEncryptionKey, ivParameterSpec); + this.delegatedBlobContainer.writeBlob(blobName, new CipherInputStream(inputStream, cipher), blobSize + GCM_TAG_BYTES_LENGTH, + failIfAlreadyExists); + } catch (NoSuchAlgorithmException | InvalidKeyException | NoSuchPaddingException | IllegalBlockSizeException + | InvalidAlgorithmParameterException e) { + throw new IOException(e); + } + } + + @Override + public void writeBlobAtomic(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) + throws IOException { + // does not support atomic write + writeBlob(blobName, inputStream, blobSize, failIfAlreadyExists); + } + + @Override + public void deleteBlob(String blobName) throws IOException { + this.delegatedBlobContainer.deleteBlob(blobName); + this.encryptionMetadataBlobContainer.deleteBlob(blobName); + } + + @Override + public DeleteResult delete() throws IOException { + DeleteResult result = this.delegatedBlobContainer.delete(); + this.encryptionMetadataBlobContainer.delete(); + return result; + } + + @Override + public Map listBlobs() throws IOException { + return this.delegatedBlobContainer.listBlobs(); + } + + @Override + public Map children() throws IOException { + return this.delegatedBlobContainer.children(); + } + + @Override + public Map listBlobsByPrefix(String blobNamePrefix) throws IOException { + Map delegatedBlobs = this.delegatedBlobContainer.listBlobsByPrefix(blobNamePrefix); + Map delegatedBlobsWithPlainSize = new HashMap<>(delegatedBlobs.size()); + for (Map.Entry entry : delegatedBlobs.entrySet()) { + delegatedBlobsWithPlainSize.put(entry.getKey(), new BlobMetaData() { + + @Override + public String name() { + return entry.getValue().name(); + } + + @Override + public long length() { + return entry.getValue().length() - GCM_TAG_BYTES_LENGTH; + } + }); + } + return delegatedBlobsWithPlainSize; + } + } + + private static SecretKey generateSecretKeyFromPassword(char[] password) throws NoSuchAlgorithmException, InvalidKeySpecException { + byte[] salt = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; // same salt for 1:1 password to key + PBEKeySpec spec = new PBEKeySpec(password, salt, 65536, 256); + SecretKey tmp = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA256").generateSecret(spec); + return new SecretKeySpec(tmp.getEncoded(), "AES"); + } + + private static String keyId(SecretKey secretKey) { + return MessageDigests.toHexString(MessageDigests.sha256().digest(secretKey.getEncoded())); + } + + private static SecretKey generateRandomSecretKey() throws NoSuchAlgorithmException { + KeyGenerator keyGen = KeyGenerator.getInstance("AES", BC_FIPS_PROV); + keyGen.init(256); + return keyGen.generateKey(); + } + + private static byte[] wrapKey(SecretKey toWrap, SecretKey keyWrappingKey) + throws NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException { + Cipher cipher = Cipher.getInstance("AESWrap"); + cipher.init(Cipher.WRAP_MODE, keyWrappingKey); + return cipher.wrap(toWrap); + } + + private static SecretKey unwrapKey(byte[] toUnwrap, SecretKey keyEncryptionKey) + throws NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException { + Cipher cipher = Cipher.getInstance("AESWrap"); + cipher.init(Cipher.UNWRAP_MODE, keyEncryptionKey); + return (SecretKey) cipher.unwrap(toUnwrap, "AES", Cipher.SECRET_KEY); + } +} diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepositoryPlugin.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepositoryPlugin.java new file mode 100644 index 0000000000000..c2631cdafac18 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepositoryPlugin.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.repositories.encrypted; + +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.ReloadablePlugin; +import org.elasticsearch.plugins.RepositoryPlugin; +import org.elasticsearch.repositories.Repository; +import org.elasticsearch.threadpool.ThreadPool; + +import java.util.List; +import java.util.Map; + +public class EncryptedRepositoryPlugin extends Plugin implements RepositoryPlugin, ReloadablePlugin { + + private final Repository.Factory encryptedRepositoryFactory; + + public EncryptedRepositoryPlugin(final Settings settings) { + encryptedRepositoryFactory = EncryptedRepository.newRepositoryFactory(settings); + } + + @Override + public Map getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry, + ThreadPool threadPool) { + return Map.of("encrypted", encryptedRepositoryFactory); + } + + @Override + public List> getSettings() { + return List.of(EncryptedRepository.ENCRYPTION_PASSWORD_SETTING); + } + + @Override + public void reload(Settings settings) { + // Secure settings should be readable inside this method. + } +} diff --git a/x-pack/plugin/repository-encrypted/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/repository-encrypted/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 0000000000000..2c9870c44dddf --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +grant { + permission java.security.SecurityPermission "putProviderProperty.BC"; +}; diff --git a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/EncryptedRepositoryTests.java b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/EncryptedRepositoryTests.java new file mode 100644 index 0000000000000..e65120d749fcd --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/EncryptedRepositoryTests.java @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.repositories.encrypted; + +import org.elasticsearch.test.ESTestCase; + +public class EncryptedRepositoryTests extends ESTestCase { + public void testThatDoesNothing() { + } +} diff --git a/x-pack/plugin/repository-encrypted/src/test/resources/rest-api-spec/test/repository_encrypted/10_basic.yml b/x-pack/plugin/repository-encrypted/src/test/resources/rest-api-spec/test/repository_encrypted/10_basic.yml new file mode 100644 index 0000000000000..858ba3e21e3ae --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/test/resources/rest-api-spec/test/repository_encrypted/10_basic.yml @@ -0,0 +1,16 @@ +# Integration tests for repository-encrypted +# +"Plugin repository-encrypted is loaded": + - skip: + reason: "contains is a newly added assertion" + features: contains + - do: + cluster.state: {} + + # Get master node id + - set: { master_node: master } + + - do: + nodes.info: {} + + - contains: { nodes.$master.plugins: { name: repository-encrypted } } From 236aa79646c27f8a1fb0be7c26e36ca2f817a7d2 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Sun, 13 Oct 2019 15:30:09 +0300 Subject: [PATCH 02/29] Done Encryption without mark/reset --- .../encrypted/EncryptedRepository.java | 6 +- .../GCMPacketsEncryptedInputStream.java | 231 ++++++++++++++++++ .../encrypted/GCMPacketsMetadata.java | 13 + 3 files changed, 247 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptedInputStream.java create mode 100644 x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsMetadata.java diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java index bb1317d73260d..21632d3da7075 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java @@ -33,7 +33,7 @@ import javax.crypto.NoSuchPaddingException; import javax.crypto.SecretKey; import javax.crypto.SecretKeyFactory; -import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.GCMParameterSpec; import javax.crypto.spec.PBEKeySpec; import javax.crypto.spec.SecretKeySpec; import java.io.ByteArrayInputStream; @@ -57,8 +57,7 @@ public class EncryptedRepository extends BlobStoreRepository { private static final String ENCRYPTION_MODE = "AES/GCM/NoPadding"; private static final String ENCRYPTION_METADATA_PREFIX = "encryption-metadata-"; // always the same IV because the key is randomly generated anew (Key-IV pair is never repeated) - //private static final GCMParameterSpec gcmParameterSpec = new GCMParameterSpec(128, new byte[] {0,1,2,3,4,5,6,7,8,9,10,11 }); - private static final IvParameterSpec ivParameterSpec = new IvParameterSpec(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }); + private static final GCMParameterSpec gcmParameterSpec = new GCMParameterSpec(128, new byte[] {0,1,2,3,4,5,6,7,8,9,10,11 }); // given the mode, the IV and the tag length, the maximum "chunk" size is ~64GB, we set it to 32GB to err on the safe side public static final ByteSizeValue MAX_CHUNK_SIZE = new ByteSizeValue(32, ByteSizeUnit.GB); @@ -215,6 +214,7 @@ public void writeBlob(String blobName, InputStream inputStream, long blobSize, b } Cipher cipher = Cipher.getInstance(ENCRYPTION_MODE, BC_FIPS_PROV); cipher.init(Cipher.ENCRYPT_MODE, dataEncryptionKey, ivParameterSpec); + cipher.update() this.delegatedBlobContainer.writeBlob(blobName, new CipherInputStream(inputStream, cipher), blobSize + GCM_TAG_BYTES_LENGTH, failIfAlreadyExists); } catch (NoSuchAlgorithmException | InvalidKeyException | NoSuchPaddingException | IllegalBlockSizeException diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptedInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptedInputStream.java new file mode 100644 index 0000000000000..aa952e603cc56 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptedInputStream.java @@ -0,0 +1,231 @@ +package org.elasticsearch.repositories.encrypted; + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.SecretKey; +import javax.crypto.ShortBufferException; +import javax.crypto.spec.GCMParameterSpec; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.security.GeneralSecurityException; +import java.security.Provider; +import java.security.SecureRandom; + +import static javax.crypto.Cipher.ENCRYPT_MODE; + +/** + * This is NOT thread-safe. + */ +public class GCMPacketsEncryptedInputStream extends FilterInputStream { + + private static final int GCM_TAG_SIZE_IN_BYTES = 16; + private static final int GCM_IV_SIZE_IN_BYTES = 12; + private static final String GCM_ENCRYPTION_MODE = "AES/GCM/NoPadding"; + + private static final int PACKET_SIZE_IN_BYTES = 4096; + private static final int READ_BUFFER_SIZE_IN_BYTES = 512; + + private boolean done = false; + private boolean closed = false; + private final Provider cipherSecurityProvider; + private final SecretKey encryptionKey; + private Cipher packetCipher; + + private long runningPacketIndex; + private final ByteBuffer runningPacketIV; + private int roomLeftInPacket; + + private byte[] plaintextBuffer = new byte[READ_BUFFER_SIZE_IN_BYTES]; + private byte[] ciphertextBuffer = new byte[READ_BUFFER_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES]; + private int ciphertextStartOffset = 0; + private int ciphertextEndOffset = 0; + private int readButNotEncrypted = 0; + + public GCMPacketsEncryptedInputStream(InputStream in, Provider provider, SecretKey secretKey) throws GeneralSecurityException { + super(in); + cipherSecurityProvider = provider; + encryptionKey = secretKey; + runningPacketIndex = 0L; + runningPacketIV = ByteBuffer.allocate(GCM_IV_SIZE_IN_BYTES); + // the first 8 bytes of the IV for packet encryption are the index of the packet + runningPacketIV.putLong(runningPacketIndex); + // the last 4 bytes of the IV for packet encryption are all equal (randomly generated) + runningPacketIV.putInt(new SecureRandom().nextInt()); + // how much to read from the underlying stream before finishing the current packet and starting the next one + roomLeftInPacket = PACKET_SIZE_IN_BYTES; + initCipher(); + } + + private void initCipher() throws GeneralSecurityException { + Cipher cipher; + if (cipherSecurityProvider != null) { + cipher = Cipher.getInstance(GCM_ENCRYPTION_MODE, cipherSecurityProvider); + } else { + cipher = Cipher.getInstance(GCM_ENCRYPTION_MODE); + } + runningPacketIV.putLong(0, runningPacketIndex++); + GCMParameterSpec gcmParameterSpec = new GCMParameterSpec(GCM_TAG_SIZE_IN_BYTES * Byte.SIZE, runningPacketIV.array()); + cipher.init(ENCRYPT_MODE, encryptionKey, gcmParameterSpec); + packetCipher = cipher; + } + + private int readAndEncrypt() throws IOException, GeneralSecurityException { + if (ciphertextStartOffset > ciphertextEndOffset) { + throw new IllegalStateException(); + } + if (ciphertextStartOffset < ciphertextEndOffset) { + // do not read anything more, there is still ciphertext to be consumed + return ciphertextEndOffset - ciphertextStartOffset; + } + if (done) { + return -1; + } + int bytesToRead = Math.min(plaintextBuffer.length - readButNotEncrypted, roomLeftInPacket); + if (bytesToRead <= 0) { + throw new IllegalStateException(); + } + int bytesRead = in.read(plaintextBuffer, 0, bytesToRead); + assert bytesRead != 0 : "read must return at least one byte"; + assert ciphertextEndOffset - ciphertextStartOffset == 0 : "there exists ciphertext still to be consumed, but it shouldn't"; + final int ciphertextLen; + if (bytesRead == -1) { + // end of the underlying stream to be encrypted + done = true; + try { + ciphertextLen = packetCipher.doFinal(ciphertextBuffer, 0); + } catch (ShortBufferException e) { + throw new IllegalStateException(); + } + // there should be no internally buffered (by the cipher) data remaining after doFinal + readButNotEncrypted -= ciphertextLen; + readButNotEncrypted += GCM_TAG_SIZE_IN_BYTES; + if (readButNotEncrypted != 0) { + throw new IllegalStateException(); + } + } else { + roomLeftInPacket -= bytesRead; + if (roomLeftInPacket < 0) { + throw new IllegalStateException(); + } + if (roomLeftInPacket == 0) { + // this is the last encryption for this packet + try { + ciphertextLen = packetCipher.doFinal(plaintextBuffer, 0, bytesRead, ciphertextBuffer, 0); + } catch (ShortBufferException e) { + throw new IllegalStateException(e); + } + // there should be no internally buffered (by the cipher) data remaining after doFinal + readButNotEncrypted += (bytesRead - ciphertextLen); + readButNotEncrypted += GCM_TAG_SIZE_IN_BYTES; + if (readButNotEncrypted != 0) { + throw new IllegalArgumentException(); + } + // reset the packet size for the next packet + roomLeftInPacket = PACKET_SIZE_IN_BYTES; + // reinit cipher for the next packet + initCipher(); + } else { + // this is a partial encryption inside the packet + try { + ciphertextLen = packetCipher.update(plaintextBuffer, 0, bytesRead, ciphertextBuffer, 0); + } catch (ShortBufferException e) { + throw new IllegalStateException(e); + } + // the cipher might encrypt only part of the plaintext and cache the rest + readButNotEncrypted += (bytesRead - ciphertextLen); + } + } + ciphertextStartOffset = 0; + ciphertextEndOffset = ciphertextLen; + return ciphertextLen; + } + + @Override + public int read() throws IOException { + while (ciphertextStartOffset >= ciphertextEndOffset) { + int cipherBytesAvailable = 0; + try { + cipherBytesAvailable = readAndEncrypt(); + } catch (GeneralSecurityException e) { + throw new IOException(e); + } + if (cipherBytesAvailable == -1) { + return -1; + } + } + return ciphertextBuffer[ciphertextStartOffset++]; + } + + @Override + public int read(byte b[], int off, int len) throws IOException { + int cipherBytesAvailable = 0; + while (ciphertextStartOffset >= ciphertextEndOffset) { + try { + cipherBytesAvailable = readAndEncrypt(); + } catch (GeneralSecurityException e) { + throw new IOException(e); + } + if (cipherBytesAvailable == -1) { + return -1; + } + } + if (len <= 0) { + return 0; + } + assert cipherBytesAvailable == (ciphertextEndOffset - ciphertextStartOffset); + int cipherBytesRead = Math.min(len, cipherBytesAvailable); + if (b != null) { + System.arraycopy(ciphertextBuffer, ciphertextStartOffset, b, off, cipherBytesRead); + } + ciphertextStartOffset += cipherBytesRead; + return cipherBytesRead; + } + + @Override + public long skip(long n) throws IOException { + int cipherBytesAvailable = ciphertextEndOffset - ciphertextStartOffset; + long cipherBytesSkipped = Math.min(cipherBytesAvailable, n); + if (n < 0) { + return 0; + } + ciphertextStartOffset = Math.addExact(ciphertextStartOffset, Math.toIntExact(cipherBytesSkipped)); + return cipherBytesSkipped; + } + + @Override + public int available() throws IOException { + return (ciphertextEndOffset - ciphertextStartOffset); + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + closed = true; + ciphertextStartOffset = 0; + ciphertextEndOffset = 0; + in.close(); + // Throw away the unprocessed data and throw no crypto exceptions. + // Normally the GCM cipher is fully readed before closing, so any authentication + // exceptions would occur while reading. + if (false == done) { + done = true; + try { + packetCipher.doFinal(); + } + catch (BadPaddingException | IllegalBlockSizeException ex) { + // Catch exceptions as the rest of the stream is unused. + } + } + } + + @Override + public boolean markSupported() { + return false; + } + +} diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsMetadata.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsMetadata.java new file mode 100644 index 0000000000000..7d17d87584dc4 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsMetadata.java @@ -0,0 +1,13 @@ +package org.elasticsearch.repositories.encrypted; + +import java.util.List; + +class GCMPacketsMetadata { + + + private int protocolVersion; + private byte[] initializationVector; + private List packetSizesInBytes; + + static +} From 26a65947862235360589fe494b09a8b9708051cc Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Mon, 14 Oct 2019 10:51:48 +0300 Subject: [PATCH 03/29] Mark in progress --- .../GCMPacketsEncryptedInputStream.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptedInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptedInputStream.java index aa952e603cc56..6231e80e037aa 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptedInputStream.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptedInputStream.java @@ -35,8 +35,9 @@ public class GCMPacketsEncryptedInputStream extends FilterInputStream { private Cipher packetCipher; private long runningPacketIndex; - private final ByteBuffer runningPacketIV; - private int roomLeftInPacket; + private final ByteBuffer runningPacketIV = ByteBuffer.allocate(GCM_IV_SIZE_IN_BYTES); + // how much to read from the underlying stream before finishing the current packet and starting the next one + private int roomLeftInPacket = PACKET_SIZE_IN_BYTES; private byte[] plaintextBuffer = new byte[READ_BUFFER_SIZE_IN_BYTES]; private byte[] ciphertextBuffer = new byte[READ_BUFFER_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES]; @@ -45,17 +46,19 @@ public class GCMPacketsEncryptedInputStream extends FilterInputStream { private int readButNotEncrypted = 0; public GCMPacketsEncryptedInputStream(InputStream in, Provider provider, SecretKey secretKey) throws GeneralSecurityException { + this(in, provider, secretKey, 0, new SecureRandom().nextInt()); + } + + private GCMPacketsEncryptedInputStream(InputStream in, Provider provider, SecretKey secretKey, long packetIndex, int nonce) + throws GeneralSecurityException { super(in); cipherSecurityProvider = provider; encryptionKey = secretKey; - runningPacketIndex = 0L; - runningPacketIV = ByteBuffer.allocate(GCM_IV_SIZE_IN_BYTES); + runningPacketIndex = packetIndex; // the first 8 bytes of the IV for packet encryption are the index of the packet - runningPacketIV.putLong(runningPacketIndex); + runningPacketIV.putLong(packetIndex); // the last 4 bytes of the IV for packet encryption are all equal (randomly generated) - runningPacketIV.putInt(new SecureRandom().nextInt()); - // how much to read from the underlying stream before finishing the current packet and starting the next one - roomLeftInPacket = PACKET_SIZE_IN_BYTES; + runningPacketIV.putInt(nonce); initCipher(); } From f1d5a84fcf4c02173f4a059b1d442b08bd1eb5c2 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Mon, 14 Oct 2019 15:00:06 +0300 Subject: [PATCH 04/29] Mark reset maybe works???? --- .../GCMPacketsEncryptedInputStream.java | 159 +++++++++++------- 1 file changed, 99 insertions(+), 60 deletions(-) diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptedInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptedInputStream.java index 6231e80e037aa..78ecaf63a6639 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptedInputStream.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptedInputStream.java @@ -6,6 +6,8 @@ import javax.crypto.SecretKey; import javax.crypto.ShortBufferException; import javax.crypto.spec.GCMParameterSpec; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; @@ -41,16 +43,20 @@ public class GCMPacketsEncryptedInputStream extends FilterInputStream { private byte[] plaintextBuffer = new byte[READ_BUFFER_SIZE_IN_BYTES]; private byte[] ciphertextBuffer = new byte[READ_BUFFER_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES]; - private int ciphertextStartOffset = 0; - private int ciphertextEndOffset = 0; - private int readButNotEncrypted = 0; + private ByteArrayInputStream ciphertextReadBuffer = new ByteArrayInputStream(new byte[0]); + private int bytesBufferedInsideTheCipher = 0; - public GCMPacketsEncryptedInputStream(InputStream in, Provider provider, SecretKey secretKey) throws GeneralSecurityException { + private ByteArrayOutputStream markWriteBuffer = null; + private ByteArrayInputStream markReadBuffer = new ByteArrayInputStream(new byte[0]); + private boolean markTriggered = false; + private long markPacketIndex; + private int markReadLimit; + + public GCMPacketsEncryptedInputStream(InputStream in, Provider provider, SecretKey secretKey) { this(in, provider, secretKey, 0, new SecureRandom().nextInt()); } - private GCMPacketsEncryptedInputStream(InputStream in, Provider provider, SecretKey secretKey, long packetIndex, int nonce) - throws GeneralSecurityException { + private GCMPacketsEncryptedInputStream(InputStream in, Provider provider, SecretKey secretKey, long packetIndex, int nonce) { super(in); cipherSecurityProvider = provider; encryptionKey = secretKey; @@ -59,7 +65,6 @@ private GCMPacketsEncryptedInputStream(InputStream in, Provider provider, Secret runningPacketIV.putLong(packetIndex); // the last 4 bytes of the IV for packet encryption are all equal (randomly generated) runningPacketIV.putInt(nonce); - initCipher(); } private void initCipher() throws GeneralSecurityException { @@ -76,23 +81,24 @@ private void initCipher() throws GeneralSecurityException { } private int readAndEncrypt() throws IOException, GeneralSecurityException { - if (ciphertextStartOffset > ciphertextEndOffset) { - throw new IllegalStateException(); - } - if (ciphertextStartOffset < ciphertextEndOffset) { - // do not read anything more, there is still ciphertext to be consumed - return ciphertextEndOffset - ciphertextStartOffset; + // do not read anything more, there is still ciphertext to be consumed + if (ciphertextReadBuffer.available() > 0) { + ciphertextReadBuffer.available(); } + // the underlying input stream is exhausted if (done) { return -1; } - int bytesToRead = Math.min(plaintextBuffer.length - readButNotEncrypted, roomLeftInPacket); + if (roomLeftInPacket == PACKET_SIZE_IN_BYTES) { + readAtTheStartOfPacket(); + } + int bytesToRead = Math.min(plaintextBuffer.length - bytesBufferedInsideTheCipher, roomLeftInPacket); if (bytesToRead <= 0) { throw new IllegalStateException(); } int bytesRead = in.read(plaintextBuffer, 0, bytesToRead); assert bytesRead != 0 : "read must return at least one byte"; - assert ciphertextEndOffset - ciphertextStartOffset == 0 : "there exists ciphertext still to be consumed, but it shouldn't"; + assert ciphertextReadBuffer.available() == 0 : "there exists ciphertext still to be consumed, but it shouldn't"; final int ciphertextLen; if (bytesRead == -1) { // end of the underlying stream to be encrypted @@ -103,9 +109,9 @@ private int readAndEncrypt() throws IOException, GeneralSecurityException { throw new IllegalStateException(); } // there should be no internally buffered (by the cipher) data remaining after doFinal - readButNotEncrypted -= ciphertextLen; - readButNotEncrypted += GCM_TAG_SIZE_IN_BYTES; - if (readButNotEncrypted != 0) { + bytesBufferedInsideTheCipher -= ciphertextLen; + bytesBufferedInsideTheCipher += GCM_TAG_SIZE_IN_BYTES; + if (bytesBufferedInsideTheCipher != 0) { throw new IllegalStateException(); } } else { @@ -121,15 +127,13 @@ private int readAndEncrypt() throws IOException, GeneralSecurityException { throw new IllegalStateException(e); } // there should be no internally buffered (by the cipher) data remaining after doFinal - readButNotEncrypted += (bytesRead - ciphertextLen); - readButNotEncrypted += GCM_TAG_SIZE_IN_BYTES; - if (readButNotEncrypted != 0) { + bytesBufferedInsideTheCipher += (bytesRead - ciphertextLen); + bytesBufferedInsideTheCipher += GCM_TAG_SIZE_IN_BYTES; + if (bytesBufferedInsideTheCipher != 0) { throw new IllegalArgumentException(); } // reset the packet size for the next packet roomLeftInPacket = PACKET_SIZE_IN_BYTES; - // reinit cipher for the next packet - initCipher(); } else { // this is a partial encryption inside the packet try { @@ -138,69 +142,77 @@ private int readAndEncrypt() throws IOException, GeneralSecurityException { throw new IllegalStateException(e); } // the cipher might encrypt only part of the plaintext and cache the rest - readButNotEncrypted += (bytesRead - ciphertextLen); + bytesBufferedInsideTheCipher += (bytesRead - ciphertextLen); } } - ciphertextStartOffset = 0; - ciphertextEndOffset = ciphertextLen; + ciphertextReadBuffer = new ByteArrayInputStream(ciphertextBuffer, 0, ciphertextLen); return ciphertextLen; } @Override public int read() throws IOException { - while (ciphertextStartOffset >= ciphertextEndOffset) { - int cipherBytesAvailable = 0; + // first try read from the buffered bytes after the mark inside the packet + if (markReadBuffer.available() > 0) { + return markReadBuffer.read(); + } + while (ciphertextReadBuffer.available() <= 0) { try { - cipherBytesAvailable = readAndEncrypt(); + if (readAndEncrypt() == -1) { + return -1; + } } catch (GeneralSecurityException e) { throw new IOException(e); } - if (cipherBytesAvailable == -1) { - return -1; - } } - return ciphertextBuffer[ciphertextStartOffset++]; + int cipherByte = ciphertextReadBuffer.read(); + if (markTriggered && cipherByte != -1) { + markWriteBuffer.write(cipherByte); + } + return cipherByte; } @Override public int read(byte b[], int off, int len) throws IOException { - int cipherBytesAvailable = 0; - while (ciphertextStartOffset >= ciphertextEndOffset) { + int bytesReadFromMarkBuffer = markReadBuffer.read(b, off, len); + if (bytesReadFromMarkBuffer != -1) { + return bytesReadFromMarkBuffer; + } + while (ciphertextReadBuffer.available() <= 0) { try { - cipherBytesAvailable = readAndEncrypt(); + if( readAndEncrypt() == -1) { + return -1; + } } catch (GeneralSecurityException e) { throw new IOException(e); } - if (cipherBytesAvailable == -1) { - return -1; - } - } - if (len <= 0) { - return 0; } - assert cipherBytesAvailable == (ciphertextEndOffset - ciphertextStartOffset); - int cipherBytesRead = Math.min(len, cipherBytesAvailable); - if (b != null) { - System.arraycopy(ciphertextBuffer, ciphertextStartOffset, b, off, cipherBytesRead); + int bytesReadFromCipherBuffer = ciphertextReadBuffer.read(b, off, len); + if (markTriggered && bytesReadFromCipherBuffer != -1) { + markWriteBuffer.write(b, off, bytesReadFromCipherBuffer); } - ciphertextStartOffset += cipherBytesRead; - return cipherBytesRead; + return bytesReadFromCipherBuffer; } @Override public long skip(long n) throws IOException { - int cipherBytesAvailable = ciphertextEndOffset - ciphertextStartOffset; - long cipherBytesSkipped = Math.min(cipherBytesAvailable, n); - if (n < 0) { - return 0; + if (markReadBuffer.available() > 0) { + return markReadBuffer.skip(n); + } + if (markTriggered) { + ciphertextReadBuffer.mark(PACKET_SIZE_IN_BYTES); + int skipAheadBytes = Math.toIntExact(ciphertextReadBuffer.skip(n)); + byte[] temp = new byte[skipAheadBytes]; + ciphertextReadBuffer.read(temp); + markWriteBuffer.write(temp); + return skipAheadBytes; + } else { + return ciphertextReadBuffer.skip(n); } - ciphertextStartOffset = Math.addExact(ciphertextStartOffset, Math.toIntExact(cipherBytesSkipped)); - return cipherBytesSkipped; } @Override public int available() throws IOException { - return (ciphertextEndOffset - ciphertextStartOffset); + return markReadBuffer.available() + ciphertextReadBuffer.available(); } @Override @@ -209,8 +221,7 @@ public void close() throws IOException { return; } closed = true; - ciphertextStartOffset = 0; - ciphertextEndOffset = 0; + ciphertextReadBuffer = new ByteArrayInputStream(new byte[0]); in.close(); // Throw away the unprocessed data and throw no crypto exceptions. // Normally the GCM cipher is fully readed before closing, so any authentication @@ -219,8 +230,7 @@ public void close() throws IOException { done = true; try { packetCipher.doFinal(); - } - catch (BadPaddingException | IllegalBlockSizeException ex) { + } catch (BadPaddingException | IllegalBlockSizeException ex) { // Catch exceptions as the rest of the stream is unused. } } @@ -228,7 +238,36 @@ public void close() throws IOException { @Override public boolean markSupported() { - return false; + return in.markSupported(); } + @Override + public void mark(int readLimit) { + markTriggered = true; + markWriteBuffer = new ByteArrayOutputStream(); + markPacketIndex = runningPacketIndex; + markReadLimit = readLimit; + } + + @Override + public void reset() throws IOException { + if (markWriteBuffer == null) { + throw new IOException(); + } + if (false == markTriggered) { + in.reset(); + ciphertextReadBuffer = new ByteArrayInputStream(new byte[0]); + } + markReadBuffer = new ByteArrayInputStream(markWriteBuffer.toByteArray()); + runningPacketIndex = markPacketIndex; + } + + private void readAtTheStartOfPacket() throws GeneralSecurityException { + // reinit cipher for the next packet + initCipher(); + if (markTriggered) { + markTriggered = false; + in.mark(markReadLimit); + } + } } From be0cc44c64ac4424509c944f3e1815b088062a59 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Mon, 14 Oct 2019 17:06:56 +0300 Subject: [PATCH 05/29] Looking good, ready to fire up tests --- .../encrypted/EncryptedRepository.java | 54 +++---- ....java => GCMPacketsCipherInputStream.java} | 148 ++++++++++-------- .../encrypted/GCMPacketsMetadata.java | 13 -- .../GCMPacketsCipherInputStreamTest.java | 60 +++++++ 4 files changed, 170 insertions(+), 105 deletions(-) rename x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/{GCMPacketsEncryptedInputStream.java => GCMPacketsCipherInputStream.java} (58%) delete mode 100644 x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsMetadata.java create mode 100644 x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTest.java diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java index 21632d3da7075..db0b719014c14 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java @@ -14,9 +14,7 @@ import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.blobstore.DeleteResult; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.hash.MessageDigests; -import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.SecureSetting; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Setting; @@ -27,7 +25,6 @@ import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import javax.crypto.Cipher; -import javax.crypto.CipherInputStream; import javax.crypto.IllegalBlockSizeException; import javax.crypto.KeyGenerator; import javax.crypto.NoSuchPaddingException; @@ -36,10 +33,8 @@ import javax.crypto.spec.GCMParameterSpec; import javax.crypto.spec.PBEKeySpec; import javax.crypto.spec.SecretKeySpec; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.security.InvalidAlgorithmParameterException; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.security.spec.InvalidKeySpecException; @@ -193,34 +188,35 @@ public BlobPath path() { @Override public InputStream readBlob(String blobName) throws IOException { - final BytesReference dataDecryptionKeyBytes = Streams.readFully(this.encryptionMetadataBlobContainer.readBlob(blobName)); - try { - SecretKey dataDecryptionKey = unwrapKey(BytesReference.toBytes(dataDecryptionKeyBytes), this.masterSecretKey); - Cipher cipher = Cipher.getInstance(ENCRYPTION_MODE, BC_FIPS_PROV); - cipher.init(Cipher.DECRYPT_MODE, dataDecryptionKey, ivParameterSpec); - return new CipherInputStream(this.delegatedBlobContainer.readBlob(blobName), cipher); - } catch (InvalidKeyException | NoSuchAlgorithmException | NoSuchPaddingException | InvalidAlgorithmParameterException e) { - throw new IOException(e); - } +// final BytesReference dataDecryptionKeyBytes = Streams.readFully(this.encryptionMetadataBlobContainer.readBlob(blobName)); +// try { +// SecretKey dataDecryptionKey = unwrapKey(BytesReference.toBytes(dataDecryptionKeyBytes), this.masterSecretKey); +// Cipher cipher = Cipher.getInstance(ENCRYPTION_MODE, BC_FIPS_PROV); +// cipher.init(Cipher.DECRYPT_MODE, dataDecryptionKey, ivParameterSpec); +// return new CipherInputStream(this.delegatedBlobContainer.readBlob(blobName), cipher); +// } catch (InvalidKeyException | NoSuchAlgorithmException | NoSuchPaddingException | InvalidAlgorithmParameterException e) { +// throw new IOException(e); +// } + return null; } @Override public void writeBlob(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) throws IOException { - try { - SecretKey dataEncryptionKey = generateRandomSecretKey(); - byte[] wrappedDataEncryptionKey = wrapKey(dataEncryptionKey, this.masterSecretKey); - try (InputStream stream = new ByteArrayInputStream(wrappedDataEncryptionKey)) { - this.encryptionMetadataBlobContainer.writeBlob(blobName, stream, wrappedDataEncryptionKey.length, failIfAlreadyExists); - } - Cipher cipher = Cipher.getInstance(ENCRYPTION_MODE, BC_FIPS_PROV); - cipher.init(Cipher.ENCRYPT_MODE, dataEncryptionKey, ivParameterSpec); - cipher.update() - this.delegatedBlobContainer.writeBlob(blobName, new CipherInputStream(inputStream, cipher), blobSize + GCM_TAG_BYTES_LENGTH, - failIfAlreadyExists); - } catch (NoSuchAlgorithmException | InvalidKeyException | NoSuchPaddingException | IllegalBlockSizeException - | InvalidAlgorithmParameterException e) { - throw new IOException(e); - } +// try { +// SecretKey dataEncryptionKey = generateRandomSecretKey(); +// byte[] wrappedDataEncryptionKey = wrapKey(dataEncryptionKey, this.masterSecretKey); +// try (InputStream stream = new ByteArrayInputStream(wrappedDataEncryptionKey)) { +// this.encryptionMetadataBlobContainer.writeBlob(blobName, stream, wrappedDataEncryptionKey.length, failIfAlreadyExists); +// } +// Cipher cipher = Cipher.getInstance(ENCRYPTION_MODE, BC_FIPS_PROV); +// cipher.init(Cipher.ENCRYPT_MODE, dataEncryptionKey, ivParameterSpec); +// cipher.update() +// this.delegatedBlobContainer.writeBlob(blobName, new CipherInputStream(inputStream, cipher), blobSize + GCM_TAG_BYTES_LENGTH, +// failIfAlreadyExists); +// } catch (NoSuchAlgorithmException | InvalidKeyException | NoSuchPaddingException | IllegalBlockSizeException +// | InvalidAlgorithmParameterException e) { +// throw new IOException(e); +// } } @Override diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptedInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java similarity index 58% rename from x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptedInputStream.java rename to x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java index 78ecaf63a6639..f763922a4d9b1 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptedInputStream.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java @@ -16,34 +16,37 @@ import java.security.Provider; import java.security.SecureRandom; +import static javax.crypto.Cipher.DECRYPT_MODE; import static javax.crypto.Cipher.ENCRYPT_MODE; /** * This is NOT thread-safe. */ -public class GCMPacketsEncryptedInputStream extends FilterInputStream { +public class GCMPacketsCipherInputStream extends FilterInputStream { private static final int GCM_TAG_SIZE_IN_BYTES = 16; private static final int GCM_IV_SIZE_IN_BYTES = 12; - private static final String GCM_ENCRYPTION_MODE = "AES/GCM/NoPadding"; + private static final String GCM_ENCRYPTION_SCHEME = "AES/GCM/NoPadding"; private static final int PACKET_SIZE_IN_BYTES = 4096; private static final int READ_BUFFER_SIZE_IN_BYTES = 512; private boolean done = false; private boolean closed = false; - private final Provider cipherSecurityProvider; - private final SecretKey encryptionKey; + private final Provider provider; + private final SecretKey secretKey; + private final int mode; private Cipher packetCipher; - private long runningPacketIndex; - private final ByteBuffer runningPacketIV = ByteBuffer.allocate(GCM_IV_SIZE_IN_BYTES); + private long packetIndex; + private final ByteBuffer packetIV = ByteBuffer.allocate(GCM_IV_SIZE_IN_BYTES); // how much to read from the underlying stream before finishing the current packet and starting the next one - private int roomLeftInPacket = PACKET_SIZE_IN_BYTES; + private int stillToReadInPacket; + private int packetSizeInBytes; - private byte[] plaintextBuffer = new byte[READ_BUFFER_SIZE_IN_BYTES]; - private byte[] ciphertextBuffer = new byte[READ_BUFFER_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES]; - private ByteArrayInputStream ciphertextReadBuffer = new ByteArrayInputStream(new byte[0]); + private byte[] inputByteBuffer = new byte[READ_BUFFER_SIZE_IN_BYTES]; + private byte[] processedByteBuffer = new byte[READ_BUFFER_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES]; + private ByteArrayInputStream processedInputStream = new ByteArrayInputStream(new byte[0]); private int bytesBufferedInsideTheCipher = 0; private ByteArrayOutputStream markWriteBuffer = null; @@ -52,101 +55,119 @@ public class GCMPacketsEncryptedInputStream extends FilterInputStream { private long markPacketIndex; private int markReadLimit; - public GCMPacketsEncryptedInputStream(InputStream in, Provider provider, SecretKey secretKey) { - this(in, provider, secretKey, 0, new SecureRandom().nextInt()); + public static GCMPacketsCipherInputStream getGCMPacketsEncryptor(InputStream in, Provider provider, SecretKey secretKey) { + return new GCMPacketsCipherInputStream(in, provider, secretKey, ENCRYPT_MODE, 0, new SecureRandom().nextInt()); } - private GCMPacketsEncryptedInputStream(InputStream in, Provider provider, SecretKey secretKey, long packetIndex, int nonce) { + public static GCMPacketsCipherInputStream getGCMPacketsDecryptor(InputStream in, Provider provider, SecretKey secretKey) { + return new GCMPacketsCipherInputStream(in, provider, secretKey, DECRYPT_MODE, 0, new SecureRandom().nextInt()); + } + + private GCMPacketsCipherInputStream(InputStream in, Provider provider, SecretKey secretKey, int mode, long packetIndex, int nonce) { super(in); - cipherSecurityProvider = provider; - encryptionKey = secretKey; - runningPacketIndex = packetIndex; + this.provider = provider; + this.secretKey = secretKey; + this.mode = mode; + this.packetIndex = packetIndex; // the first 8 bytes of the IV for packet encryption are the index of the packet - runningPacketIV.putLong(packetIndex); + packetIV.putLong(packetIndex); // the last 4 bytes of the IV for packet encryption are all equal (randomly generated) - runningPacketIV.putInt(nonce); + packetIV.putInt(nonce); + if (mode == ENCRYPT_MODE) { + packetSizeInBytes = PACKET_SIZE_IN_BYTES; + } else { + packetSizeInBytes = PACKET_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES; + } } private void initCipher() throws GeneralSecurityException { Cipher cipher; - if (cipherSecurityProvider != null) { - cipher = Cipher.getInstance(GCM_ENCRYPTION_MODE, cipherSecurityProvider); + if (provider != null) { + cipher = Cipher.getInstance(GCM_ENCRYPTION_SCHEME, provider); } else { - cipher = Cipher.getInstance(GCM_ENCRYPTION_MODE); + cipher = Cipher.getInstance(GCM_ENCRYPTION_SCHEME); } - runningPacketIV.putLong(0, runningPacketIndex++); - GCMParameterSpec gcmParameterSpec = new GCMParameterSpec(GCM_TAG_SIZE_IN_BYTES * Byte.SIZE, runningPacketIV.array()); - cipher.init(ENCRYPT_MODE, encryptionKey, gcmParameterSpec); + packetIV.putLong(0, packetIndex++); + GCMParameterSpec gcmParameterSpec = new GCMParameterSpec(GCM_TAG_SIZE_IN_BYTES * Byte.SIZE, packetIV.array()); + cipher.init(mode, secretKey, gcmParameterSpec); packetCipher = cipher; } - private int readAndEncrypt() throws IOException, GeneralSecurityException { - // do not read anything more, there is still ciphertext to be consumed - if (ciphertextReadBuffer.available() > 0) { - ciphertextReadBuffer.available(); + private int readAndProcess() throws IOException, GeneralSecurityException { + // do not read anything more, there are still processed bytes to be consumed + if (processedInputStream.available() > 0) { + processedInputStream.available(); } // the underlying input stream is exhausted if (done) { return -1; } - if (roomLeftInPacket == PACKET_SIZE_IN_BYTES) { + if (stillToReadInPacket == packetSizeInBytes) { readAtTheStartOfPacket(); } - int bytesToRead = Math.min(plaintextBuffer.length - bytesBufferedInsideTheCipher, roomLeftInPacket); + int bytesToRead = Math.min(inputByteBuffer.length - bytesBufferedInsideTheCipher, stillToReadInPacket); if (bytesToRead <= 0) { throw new IllegalStateException(); } - int bytesRead = in.read(plaintextBuffer, 0, bytesToRead); + int bytesRead = in.read(inputByteBuffer, 0, bytesToRead); assert bytesRead != 0 : "read must return at least one byte"; - assert ciphertextReadBuffer.available() == 0 : "there exists ciphertext still to be consumed, but it shouldn't"; - final int ciphertextLen; + assert processedInputStream.available() == 0 : "there exists processed still to be consumed, but it shouldn't"; + final int bytesProcessed; if (bytesRead == -1) { // end of the underlying stream to be encrypted done = true; try { - ciphertextLen = packetCipher.doFinal(ciphertextBuffer, 0); + bytesProcessed = packetCipher.doFinal(processedByteBuffer, 0); } catch (ShortBufferException e) { throw new IllegalStateException(); } // there should be no internally buffered (by the cipher) data remaining after doFinal - bytesBufferedInsideTheCipher -= ciphertextLen; - bytesBufferedInsideTheCipher += GCM_TAG_SIZE_IN_BYTES; + bytesBufferedInsideTheCipher -= bytesProcessed; + if (mode == ENCRYPT_MODE) { + bytesBufferedInsideTheCipher += GCM_TAG_SIZE_IN_BYTES; + } else { + bytesBufferedInsideTheCipher -= GCM_TAG_SIZE_IN_BYTES; + } if (bytesBufferedInsideTheCipher != 0) { throw new IllegalStateException(); } } else { - roomLeftInPacket -= bytesRead; - if (roomLeftInPacket < 0) { + stillToReadInPacket -= bytesRead; + if (stillToReadInPacket < 0) { throw new IllegalStateException(); } - if (roomLeftInPacket == 0) { + if (stillToReadInPacket == 0) { // this is the last encryption for this packet try { - ciphertextLen = packetCipher.doFinal(plaintextBuffer, 0, bytesRead, ciphertextBuffer, 0); + bytesProcessed = packetCipher.doFinal(inputByteBuffer, 0, bytesRead, processedByteBuffer, 0); } catch (ShortBufferException e) { throw new IllegalStateException(e); } // there should be no internally buffered (by the cipher) data remaining after doFinal - bytesBufferedInsideTheCipher += (bytesRead - ciphertextLen); - bytesBufferedInsideTheCipher += GCM_TAG_SIZE_IN_BYTES; + bytesBufferedInsideTheCipher += (bytesRead - bytesProcessed); + if (mode == ENCRYPT_MODE) { + bytesBufferedInsideTheCipher += GCM_TAG_SIZE_IN_BYTES; + } else { + bytesBufferedInsideTheCipher -= GCM_TAG_SIZE_IN_BYTES; + } if (bytesBufferedInsideTheCipher != 0) { throw new IllegalArgumentException(); } // reset the packet size for the next packet - roomLeftInPacket = PACKET_SIZE_IN_BYTES; + stillToReadInPacket = packetSizeInBytes; } else { // this is a partial encryption inside the packet try { - ciphertextLen = packetCipher.update(plaintextBuffer, 0, bytesRead, ciphertextBuffer, 0); + bytesProcessed = packetCipher.update(inputByteBuffer, 0, bytesRead, processedByteBuffer, 0); } catch (ShortBufferException e) { throw new IllegalStateException(e); } // the cipher might encrypt only part of the plaintext and cache the rest - bytesBufferedInsideTheCipher += (bytesRead - ciphertextLen); + bytesBufferedInsideTheCipher += (bytesRead - bytesProcessed); } } - ciphertextReadBuffer = new ByteArrayInputStream(ciphertextBuffer, 0, ciphertextLen); - return ciphertextLen; + processedInputStream = new ByteArrayInputStream(processedByteBuffer, 0, bytesProcessed); + return bytesProcessed; } @Override @@ -155,16 +176,16 @@ public int read() throws IOException { if (markReadBuffer.available() > 0) { return markReadBuffer.read(); } - while (ciphertextReadBuffer.available() <= 0) { + while (processedInputStream.available() <= 0) { try { - if (readAndEncrypt() == -1) { + if (readAndProcess() == -1) { return -1; } } catch (GeneralSecurityException e) { throw new IOException(e); } } - int cipherByte = ciphertextReadBuffer.read(); + int cipherByte = processedInputStream.read(); if (markTriggered && cipherByte != -1) { markWriteBuffer.write(cipherByte); } @@ -177,16 +198,16 @@ public int read(byte b[], int off, int len) throws IOException { if (bytesReadFromMarkBuffer != -1) { return bytesReadFromMarkBuffer; } - while (ciphertextReadBuffer.available() <= 0) { + while (processedInputStream.available() <= 0) { try { - if( readAndEncrypt() == -1) { + if( readAndProcess() == -1) { return -1; } } catch (GeneralSecurityException e) { throw new IOException(e); } } - int bytesReadFromCipherBuffer = ciphertextReadBuffer.read(b, off, len); + int bytesReadFromCipherBuffer = processedInputStream.read(b, off, len); if (markTriggered && bytesReadFromCipherBuffer != -1) { markWriteBuffer.write(b, off, bytesReadFromCipherBuffer); } @@ -199,20 +220,20 @@ public long skip(long n) throws IOException { return markReadBuffer.skip(n); } if (markTriggered) { - ciphertextReadBuffer.mark(PACKET_SIZE_IN_BYTES); - int skipAheadBytes = Math.toIntExact(ciphertextReadBuffer.skip(n)); + processedInputStream.mark(packetSizeInBytes); + int skipAheadBytes = Math.toIntExact(processedInputStream.skip(n)); byte[] temp = new byte[skipAheadBytes]; - ciphertextReadBuffer.read(temp); + processedInputStream.read(temp); markWriteBuffer.write(temp); return skipAheadBytes; } else { - return ciphertextReadBuffer.skip(n); + return processedInputStream.skip(n); } } @Override public int available() throws IOException { - return markReadBuffer.available() + ciphertextReadBuffer.available(); + return markReadBuffer.available() + processedInputStream.available(); } @Override @@ -221,7 +242,7 @@ public void close() throws IOException { return; } closed = true; - ciphertextReadBuffer = new ByteArrayInputStream(new byte[0]); + processedInputStream = new ByteArrayInputStream(new byte[0]); in.close(); // Throw away the unprocessed data and throw no crypto exceptions. // Normally the GCM cipher is fully readed before closing, so any authentication @@ -245,7 +266,7 @@ public boolean markSupported() { public void mark(int readLimit) { markTriggered = true; markWriteBuffer = new ByteArrayOutputStream(); - markPacketIndex = runningPacketIndex; + markPacketIndex = packetIndex; markReadLimit = readLimit; } @@ -256,17 +277,18 @@ public void reset() throws IOException { } if (false == markTriggered) { in.reset(); - ciphertextReadBuffer = new ByteArrayInputStream(new byte[0]); + processedInputStream = new ByteArrayInputStream(new byte[0]); } markReadBuffer = new ByteArrayInputStream(markWriteBuffer.toByteArray()); - runningPacketIndex = markPacketIndex; + packetIndex = markPacketIndex; } private void readAtTheStartOfPacket() throws GeneralSecurityException { - // reinit cipher for the next packet + // re init cipher for this following packet initCipher(); if (markTriggered) { markTriggered = false; + // mark the underlying stream at the start of the packet in.mark(markReadLimit); } } diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsMetadata.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsMetadata.java deleted file mode 100644 index 7d17d87584dc4..0000000000000 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsMetadata.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.elasticsearch.repositories.encrypted; - -import java.util.List; - -class GCMPacketsMetadata { - - - private int protocolVersion; - private byte[] initializationVector; - private List packetSizesInBytes; - - static -} diff --git a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTest.java b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTest.java new file mode 100644 index 0000000000000..b3e3cf1fba6ea --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTest.java @@ -0,0 +1,60 @@ +package org.elasticsearch.repositories.encrypted; + +import org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider; +import org.hamcrest.Matchers; +import org.junit.Assert; + +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.util.Random; + +public class GCMPacketsCipherInputStreamTest { + + private static final BouncyCastleFipsProvider BC_FIPS_PROV = new BouncyCastleFipsProvider(); + + public void testBasicEncryptDecrypt() throws Exception { + KeyGenerator keyGen = KeyGenerator.getInstance("AES", BC_FIPS_PROV); + keyGen.init(256); + SecretKey secretKey = keyGen.generateKey(); + + for (int i = 0; i < 32; i++) { + testEncryptDecryptRandomOfLength(i, secretKey); + } + } + + private void testEncryptDecryptRandomOfLength(int length, SecretKey secretKey) throws Exception { + Random random = new Random(); + byte[] temp = new byte[length]; + byte[] plaintextArray = new byte[length]; + random.nextBytes(plaintextArray); + ByteArrayOutputStream cipherTextOutput = new ByteArrayOutputStream(); + ByteArrayOutputStream plainTextOutput = new ByteArrayOutputStream(); + // encrypt + try (InputStream cipherInputStream = GCMPacketsCipherInputStream.getGCMPacketsEncryptor(new ByteArrayInputStream(plaintextArray), + BC_FIPS_PROV, secretKey)) { + do { + int bytesEncrypted = cipherInputStream.read(temp, 0, random.nextInt(temp.length)); + if (bytesEncrypted == -1) { + break; + } + cipherTextOutput.write(temp, 0, bytesEncrypted); + } while (true); + } + //decrypt + try (InputStream plainInputStream = + GCMPacketsCipherInputStream.getGCMPacketsDecryptor(new ByteArrayInputStream(cipherTextOutput.toByteArray()), + BC_FIPS_PROV, secretKey)) { + do { + int bytesDecrypted = plainInputStream.read(temp, 0, random.nextInt(temp.length)); + if (bytesDecrypted == -1) { + break; + } + plainTextOutput.write(temp, 0, bytesDecrypted); + } while (true); + } + Assert.assertThat(plainTextOutput.toByteArray(), Matchers.is(plaintextArray)); + } +} \ No newline at end of file From b647657212000f4cc210f94de0bd7e5e72029a66 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 15 Oct 2019 16:33:31 +0300 Subject: [PATCH 06/29] hardcore simple tests work! --- .../GCMPacketsCipherInputStream.java | 30 ++++++++----- .../plugin-metadata/plugin-security.policy | 5 ++- ... => GCMPacketsCipherInputStreamTests.java} | 45 ++++++++++++++----- 3 files changed, 58 insertions(+), 22 deletions(-) rename x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/{GCMPacketsCipherInputStreamTest.java => GCMPacketsCipherInputStreamTests.java} (56%) diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java index f763922a4d9b1..c407a117300d7 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java @@ -1,5 +1,7 @@ package org.elasticsearch.repositories.encrypted; +import org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider; + import javax.crypto.BadPaddingException; import javax.crypto.Cipher; import javax.crypto.IllegalBlockSizeException; @@ -14,7 +16,6 @@ import java.nio.ByteBuffer; import java.security.GeneralSecurityException; import java.security.Provider; -import java.security.SecureRandom; import static javax.crypto.Cipher.DECRYPT_MODE; import static javax.crypto.Cipher.ENCRYPT_MODE; @@ -33,11 +34,11 @@ public class GCMPacketsCipherInputStream extends FilterInputStream { private boolean done = false; private boolean closed = false; - private final Provider provider; private final SecretKey secretKey; private final int mode; - private Cipher packetCipher; + private final Provider provider; + private Cipher packetCipher; private long packetIndex; private final ByteBuffer packetIV = ByteBuffer.allocate(GCM_IV_SIZE_IN_BYTES); // how much to read from the underlying stream before finishing the current packet and starting the next one @@ -55,20 +56,28 @@ public class GCMPacketsCipherInputStream extends FilterInputStream { private long markPacketIndex; private int markReadLimit; - public static GCMPacketsCipherInputStream getGCMPacketsEncryptor(InputStream in, Provider provider, SecretKey secretKey) { - return new GCMPacketsCipherInputStream(in, provider, secretKey, ENCRYPT_MODE, 0, new SecureRandom().nextInt()); + static GCMPacketsCipherInputStream getGCMPacketsEncryptor(InputStream in, SecretKey secretKey, int nonce, Provider provider) { + return new GCMPacketsCipherInputStream(in, secretKey, ENCRYPT_MODE, 0, nonce, provider); + } + + static GCMPacketsCipherInputStream getGCMPacketsDecryptor(InputStream in, SecretKey secretKey, int nonce, Provider provider) { + return new GCMPacketsCipherInputStream(in, secretKey, DECRYPT_MODE, 0, nonce, provider); + } + + public static GCMPacketsCipherInputStream getGCMPacketsEncryptor(InputStream in, SecretKey secretKey, int nonce) { + return getGCMPacketsEncryptor(in, secretKey, nonce, new BouncyCastleFipsProvider()); } - public static GCMPacketsCipherInputStream getGCMPacketsDecryptor(InputStream in, Provider provider, SecretKey secretKey) { - return new GCMPacketsCipherInputStream(in, provider, secretKey, DECRYPT_MODE, 0, new SecureRandom().nextInt()); + public static GCMPacketsCipherInputStream getGCMPacketsDecryptor(InputStream in, SecretKey secretKey, int nonce) { + return getGCMPacketsDecryptor(in, secretKey, nonce, new BouncyCastleFipsProvider()); } - private GCMPacketsCipherInputStream(InputStream in, Provider provider, SecretKey secretKey, int mode, long packetIndex, int nonce) { + private GCMPacketsCipherInputStream(InputStream in, SecretKey secretKey, int mode, long packetIndex, int nonce, Provider provider) { super(in); - this.provider = provider; this.secretKey = secretKey; this.mode = mode; this.packetIndex = packetIndex; + this.provider = provider; // the first 8 bytes of the IV for packet encryption are the index of the packet packetIV.putLong(packetIndex); // the last 4 bytes of the IV for packet encryption are all equal (randomly generated) @@ -78,6 +87,7 @@ private GCMPacketsCipherInputStream(InputStream in, Provider provider, SecretKey } else { packetSizeInBytes = PACKET_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES; } + stillToReadInPacket = packetSizeInBytes; } private void initCipher() throws GeneralSecurityException { @@ -200,7 +210,7 @@ public int read(byte b[], int off, int len) throws IOException { } while (processedInputStream.available() <= 0) { try { - if( readAndProcess() == -1) { + if (readAndProcess() == -1) { return -1; } } catch (GeneralSecurityException e) { diff --git a/x-pack/plugin/repository-encrypted/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/repository-encrypted/src/main/plugin-metadata/plugin-security.policy index 2c9870c44dddf..9aa4d16b19a80 100644 --- a/x-pack/plugin/repository-encrypted/src/main/plugin-metadata/plugin-security.policy +++ b/x-pack/plugin/repository-encrypted/src/main/plugin-metadata/plugin-security.policy @@ -5,5 +5,8 @@ */ grant { - permission java.security.SecurityPermission "putProviderProperty.BC"; + permission java.security.SecurityPermission "putProviderProperty.BCFIPS"; + permission java.lang.RuntimePermission "accessDeclaredMembers"; + permission java.lang.RuntimePermission "accessClassInPackage.sun.security.provider"; + permission org.bouncycastle.crypto.CryptoServicesPermission "exportSecretKey"; }; diff --git a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTest.java b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java similarity index 56% rename from x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTest.java rename to x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java index b3e3cf1fba6ea..0efe99b9a849e 100644 --- a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTest.java +++ b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java @@ -1,26 +1,48 @@ package org.elasticsearch.repositories.encrypted; import org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider; +import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; import org.junit.Assert; +import org.junit.Before; import javax.crypto.KeyGenerator; import javax.crypto.SecretKey; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.InputStream; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.security.SecureRandom; import java.util.Random; -public class GCMPacketsCipherInputStreamTest { +public class GCMPacketsCipherInputStreamTests extends ESTestCase { - private static final BouncyCastleFipsProvider BC_FIPS_PROV = new BouncyCastleFipsProvider(); + private BouncyCastleFipsProvider bcFipsProvider; - public void testBasicEncryptDecrypt() throws Exception { - KeyGenerator keyGen = KeyGenerator.getInstance("AES", BC_FIPS_PROV); - keyGen.init(256); - SecretKey secretKey = keyGen.generateKey(); + @Before + public void setup() { + AccessController.doPrivileged((PrivilegedAction) () -> { + this.bcFipsProvider = new BouncyCastleFipsProvider(); +// for (Object o : this.bcFipsProvider.keySet()) { +// System.out.println(o); +// } + return null; + }); + } + + public void testHardcoreBasicEncryptDecrypt() throws Exception { + SecretKey secretKey = AccessController.doPrivileged((PrivilegedAction) () -> { + try { + KeyGenerator keyGen = KeyGenerator.getInstance("AES", bcFipsProvider); + keyGen.init(256, SecureRandom.getInstance("DEFAULT", bcFipsProvider)); + return keyGen.generateKey(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); - for (int i = 0; i < 32; i++) { + for (int i = 1; i < 4096 * 8; i++) { testEncryptDecryptRandomOfLength(i, secretKey); } } @@ -30,13 +52,14 @@ private void testEncryptDecryptRandomOfLength(int length, SecretKey secretKey) t byte[] temp = new byte[length]; byte[] plaintextArray = new byte[length]; random.nextBytes(plaintextArray); + int nonce = random.nextInt(); ByteArrayOutputStream cipherTextOutput = new ByteArrayOutputStream(); ByteArrayOutputStream plainTextOutput = new ByteArrayOutputStream(); // encrypt try (InputStream cipherInputStream = GCMPacketsCipherInputStream.getGCMPacketsEncryptor(new ByteArrayInputStream(plaintextArray), - BC_FIPS_PROV, secretKey)) { + secretKey, nonce, bcFipsProvider)) { do { - int bytesEncrypted = cipherInputStream.read(temp, 0, random.nextInt(temp.length)); + int bytesEncrypted = cipherInputStream.read(temp, 0, randomIntBetween(1, temp.length)); if (bytesEncrypted == -1) { break; } @@ -46,9 +69,9 @@ private void testEncryptDecryptRandomOfLength(int length, SecretKey secretKey) t //decrypt try (InputStream plainInputStream = GCMPacketsCipherInputStream.getGCMPacketsDecryptor(new ByteArrayInputStream(cipherTextOutput.toByteArray()), - BC_FIPS_PROV, secretKey)) { + secretKey, nonce, bcFipsProvider)) { do { - int bytesDecrypted = plainInputStream.read(temp, 0, random.nextInt(temp.length)); + int bytesDecrypted = plainInputStream.read(temp, 0, randomIntBetween(1, temp.length)); if (bytesDecrypted == -1) { break; } From 56f3c1e46b70191e1dae0e04cd1dd37924a2148a Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 15 Oct 2019 19:04:44 +0300 Subject: [PATCH 07/29] Encryption/decryption whizzes! --- .../GCMPacketsCipherInputStream.java | 28 ++++- .../GCMPacketsCipherInputStreamTests.java | 114 ++++++++++++------ 2 files changed, 104 insertions(+), 38 deletions(-) diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java index c407a117300d7..fd8ff3c0884bd 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java @@ -21,7 +21,7 @@ import static javax.crypto.Cipher.ENCRYPT_MODE; /** - * This is NOT thread-safe. + * This is obviously NOT thread-safe. */ public class GCMPacketsCipherInputStream extends FilterInputStream { @@ -29,8 +29,8 @@ public class GCMPacketsCipherInputStream extends FilterInputStream { private static final int GCM_IV_SIZE_IN_BYTES = 12; private static final String GCM_ENCRYPTION_SCHEME = "AES/GCM/NoPadding"; - private static final int PACKET_SIZE_IN_BYTES = 4096; - private static final int READ_BUFFER_SIZE_IN_BYTES = 512; + static final int PACKET_SIZE_IN_BYTES = 4096; + static final int READ_BUFFER_SIZE_IN_BYTES = 512; private boolean done = false; private boolean closed = false; @@ -72,6 +72,16 @@ public static GCMPacketsCipherInputStream getGCMPacketsDecryptor(InputStream in, return getGCMPacketsDecryptor(in, secretKey, nonce, new BouncyCastleFipsProvider()); } + public static int getEncryptionSizeFromPlainSize(int size) { + return (size / PACKET_SIZE_IN_BYTES) * (PACKET_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES) + + (size % PACKET_SIZE_IN_BYTES) + GCM_TAG_SIZE_IN_BYTES; + } + + public static int getDecryptionSizeFromCipherSize(int size) { + return (size / (PACKET_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES)) * PACKET_SIZE_IN_BYTES + + (size % (PACKET_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES)) - GCM_TAG_SIZE_IN_BYTES; + } + private GCMPacketsCipherInputStream(InputStream in, SecretKey secretKey, int mode, long packetIndex, int nonce, Provider provider) { super(in); this.secretKey = secretKey; @@ -176,7 +186,10 @@ private int readAndProcess() throws IOException, GeneralSecurityException { bytesBufferedInsideTheCipher += (bytesRead - bytesProcessed); } } - processedInputStream = new ByteArrayInputStream(processedByteBuffer, 0, bytesProcessed); + // the "if" is just an "optimization" + if (bytesProcessed != 0) { + processedInputStream = new ByteArrayInputStream(processedByteBuffer, 0, bytesProcessed); + } return bytesProcessed; } @@ -229,11 +242,18 @@ public long skip(long n) throws IOException { if (markReadBuffer.available() > 0) { return markReadBuffer.skip(n); } + // if mark is triggered bytes cannot be discarded because they might be read later if (markTriggered) { + // mark processedInputStream.mark(packetSizeInBytes); + // skip int skipAheadBytes = Math.toIntExact(processedInputStream.skip(n)); + // reset + processedInputStream.reset(); byte[] temp = new byte[skipAheadBytes]; + // re-read the skipped bytes processedInputStream.read(temp); + // cache the skipped bytes markWriteBuffer.write(temp); return skipAheadBytes; } else { diff --git a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java index 0efe99b9a849e..7c96ff57b39e6 100644 --- a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java +++ b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java @@ -3,13 +3,14 @@ import org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; -import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import javax.crypto.KeyGenerator; import javax.crypto.SecretKey; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.io.InputStream; import java.security.AccessController; import java.security.PrivilegedAction; @@ -18,12 +19,13 @@ public class GCMPacketsCipherInputStreamTests extends ESTestCase { - private BouncyCastleFipsProvider bcFipsProvider; + private static BouncyCastleFipsProvider bcFipsProvider; + private SecretKey secretKey; - @Before - public void setup() { + @BeforeClass + static void setupProvider() { AccessController.doPrivileged((PrivilegedAction) () -> { - this.bcFipsProvider = new BouncyCastleFipsProvider(); + GCMPacketsCipherInputStreamTests.bcFipsProvider = new BouncyCastleFipsProvider(); // for (Object o : this.bcFipsProvider.keySet()) { // System.out.println(o); // } @@ -31,53 +33,97 @@ public void setup() { }); } - public void testHardcoreBasicEncryptDecrypt() throws Exception { - SecretKey secretKey = AccessController.doPrivileged((PrivilegedAction) () -> { - try { - KeyGenerator keyGen = KeyGenerator.getInstance("AES", bcFipsProvider); - keyGen.init(256, SecureRandom.getInstance("DEFAULT", bcFipsProvider)); - return keyGen.generateKey(); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); + @Before + void createSecretKey() throws Exception { + secretKey = generateSecretKey(); + } - for (int i = 1; i < 4096 * 8; i++) { + public void testEncryptDecryptEmpty() throws Exception { + testEncryptDecryptRandomOfLength(0, secretKey); + } + + public void testEncryptDecryptSmallerThanBufferSize() throws Exception { + for (int i = 1; i < GCMPacketsCipherInputStream.READ_BUFFER_SIZE_IN_BYTES; i++) { testEncryptDecryptRandomOfLength(i, secretKey); } } + public void testEncryptDecryptMultipleOfBufferSize() throws Exception { + for (int i = 1; i < 10; i++) { + testEncryptDecryptRandomOfLength(i * GCMPacketsCipherInputStream.READ_BUFFER_SIZE_IN_BYTES, secretKey); + } + } + + public void testEncryptDecryptSmallerThanPacketSize() throws Exception { + for (int i = GCMPacketsCipherInputStream.READ_BUFFER_SIZE_IN_BYTES + 1; i < GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES; i++) { + testEncryptDecryptRandomOfLength(i, secretKey); + } + } + + public void testEncryptDecryptLargerThanPacketSize() throws Exception { + for (int i = GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES + 1; i < GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES * 3; i++) { + testEncryptDecryptRandomOfLength(i, secretKey); + } + } + + public void testEncryptDecryptMultipleOfPacketSize() throws Exception { + for (int i = 1; i < 10; i++) { + testEncryptDecryptRandomOfLength(i * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES, secretKey); + } + } + private void testEncryptDecryptRandomOfLength(int length, SecretKey secretKey) throws Exception { Random random = new Random(); - byte[] temp = new byte[length]; byte[] plaintextArray = new byte[length]; random.nextBytes(plaintextArray); int nonce = random.nextInt(); - ByteArrayOutputStream cipherTextOutput = new ByteArrayOutputStream(); - ByteArrayOutputStream plainTextOutput = new ByteArrayOutputStream(); + ByteArrayOutputStream cipherTextOutput; + ByteArrayOutputStream plainTextOutput; // encrypt try (InputStream cipherInputStream = GCMPacketsCipherInputStream.getGCMPacketsEncryptor(new ByteArrayInputStream(plaintextArray), secretKey, nonce, bcFipsProvider)) { - do { - int bytesEncrypted = cipherInputStream.read(temp, 0, randomIntBetween(1, temp.length)); - if (bytesEncrypted == -1) { - break; - } - cipherTextOutput.write(temp, 0, bytesEncrypted); - } while (true); + cipherTextOutput = readAllInputStream(cipherInputStream, GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length)); } //decrypt try (InputStream plainInputStream = GCMPacketsCipherInputStream.getGCMPacketsDecryptor(new ByteArrayInputStream(cipherTextOutput.toByteArray()), secretKey, nonce, bcFipsProvider)) { - do { - int bytesDecrypted = plainInputStream.read(temp, 0, randomIntBetween(1, temp.length)); - if (bytesDecrypted == -1) { + plainTextOutput = readAllInputStream(plainInputStream, + GCMPacketsCipherInputStream.getDecryptionSizeFromCipherSize(cipherTextOutput.size())); + } + assertThat(plainTextOutput.toByteArray(), Matchers.is(plaintextArray)); + } + + private SecretKey generateSecretKey() throws Exception { + return AccessController.doPrivileged((PrivilegedAction) () -> { + try { + KeyGenerator keyGen = KeyGenerator.getInstance("AES", bcFipsProvider); + keyGen.init(256, SecureRandom.getInstance("DEFAULT", bcFipsProvider)); + return keyGen.generateKey(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + private ByteArrayOutputStream readAllInputStream(InputStream inputStream, int size) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(size); + byte[] temp = new byte[randomIntBetween(1, size != 0 ? size : 1)]; + do { + int bytesRead = inputStream.read(temp, 0, randomIntBetween(1, temp.length)); + if (bytesRead == -1) { + break; + } + baos.write(temp, 0, bytesRead); + if (randomBoolean()) { + int singleByte = inputStream.read(); + if (singleByte == -1) { break; } - plainTextOutput.write(temp, 0, bytesDecrypted); - } while (true); - } - Assert.assertThat(plainTextOutput.toByteArray(), Matchers.is(plaintextArray)); + baos.write(singleByte); + } + } while (true); + assertThat(baos.size(), Matchers.is(size)); + return baos; } -} \ No newline at end of file +} From 3d462e6cbcfc212ace3b97113846e5472618e690 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Wed, 16 Oct 2019 11:14:36 +0300 Subject: [PATCH 08/29] Before separation of MarkDecorator --- .../encrypted/GCMPacketsCipherInputStream.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java index fd8ff3c0884bd..625ccb9eb85a3 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java @@ -25,6 +25,14 @@ */ public class GCMPacketsCipherInputStream extends FilterInputStream { + static class GCMPacketsMarkDecorator extends GCMPacketsCipherInputStream { + + private GCMPacketsMarkDecorator(InputStream in, SecretKey secretKey, int mode, long packetIndex, int nonce, + Provider provider) { + super(in, secretKey, mode, packetIndex, nonce, provider); + } + } + private static final int GCM_TAG_SIZE_IN_BYTES = 16; private static final int GCM_IV_SIZE_IN_BYTES = 12; private static final String GCM_ENCRYPTION_SCHEME = "AES/GCM/NoPadding"; @@ -73,11 +81,17 @@ public static GCMPacketsCipherInputStream getGCMPacketsDecryptor(InputStream in, } public static int getEncryptionSizeFromPlainSize(int size) { + if (size < 0) { + throw new IllegalArgumentException(); + } return (size / PACKET_SIZE_IN_BYTES) * (PACKET_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES) + (size % PACKET_SIZE_IN_BYTES) + GCM_TAG_SIZE_IN_BYTES; } public static int getDecryptionSizeFromCipherSize(int size) { + if (size < GCM_TAG_SIZE_IN_BYTES) { + throw new IllegalArgumentException(); + } return (size / (PACKET_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES)) * PACKET_SIZE_IN_BYTES + (size % (PACKET_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES)) - GCM_TAG_SIZE_IN_BYTES; } @@ -253,7 +267,7 @@ public long skip(long n) throws IOException { byte[] temp = new byte[skipAheadBytes]; // re-read the skipped bytes processedInputStream.read(temp); - // cache the skipped bytes + // do not discard the skipped bytes markWriteBuffer.write(temp); return skipAheadBytes; } else { From 498f77c2e8e483ca909d66a5f02f58048d5546b8 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Wed, 16 Oct 2019 15:17:42 +0300 Subject: [PATCH 09/29] Refactored mark and reset --- .../GCMPacketsCipherInputStream.java | 227 ++++++++++++------ 1 file changed, 148 insertions(+), 79 deletions(-) diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java index 625ccb9eb85a3..77b5ca2713440 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java @@ -25,12 +25,115 @@ */ public class GCMPacketsCipherInputStream extends FilterInputStream { - static class GCMPacketsMarkDecorator extends GCMPacketsCipherInputStream { + static class GCMPacketsWithMarkCipherInputStream extends GCMPacketsCipherInputStream { - private GCMPacketsMarkDecorator(InputStream in, SecretKey secretKey, int mode, long packetIndex, int nonce, - Provider provider) { + private GCMPacketsWithMarkCipherInputStream(InputStream in, SecretKey secretKey, int mode, long packetIndex, int nonce, + Provider provider) { super(in, secretKey, mode, packetIndex, nonce, provider); } + + private ByteArrayOutputStream markWriteBuffer = null; + private ByteArrayInputStream markReadBuffer = new ByteArrayInputStream(new byte[0]); + private boolean markTriggeredForCurrentPacket = false; + private long markPacketIndex; + private int markReadLimit; + + @Override + public int read() throws IOException { + ensureOpen(); + // in case this is a reseted stream that has buffered part of the ciphertext + if (markReadBuffer.available() > 0) { + return markReadBuffer.read(); + } + int cipherByte = super.read(); + // if buffering of the ciphertext is required + if (markTriggeredForCurrentPacket && cipherByte != -1) { + markWriteBuffer.write(cipherByte); + } + return cipherByte; + } + + @Override + public int read(byte b[], int off, int len) throws IOException { + ensureOpen(); + // in case this is a reseted stream that has buffered part of the ciphertext + int bytesReadFromMarkBuffer = markReadBuffer.read(b, off, len); + if (bytesReadFromMarkBuffer != -1) { + return bytesReadFromMarkBuffer; + } + int cipherBytesCount = super.read(b, off, len); + // if buffering of the ciphertext is required + if (markTriggeredForCurrentPacket && cipherBytesCount != -1) { + markWriteBuffer.write(b, off, cipherBytesCount ); + } + return cipherBytesCount; + } + + @Override + public long skip(long n) throws IOException { + ensureOpen(); + if (markReadBuffer.available() > 0) { + return markReadBuffer.skip(n); + } + int bytesAvailable = super.available(); + bytesAvailable = Math.min(bytesAvailable, Math.toIntExact(n)); + if (markTriggeredForCurrentPacket) { + byte[] temp = new byte[bytesAvailable]; + int bytesRead = super.read(temp); + markWriteBuffer.write(temp); + return bytesRead; + } else { + return super.skip(n); + } + } + + @Override + public int available() throws IOException { + ensureOpen(); + return markReadBuffer.available() + super.available(); + } + + @Override + public boolean markSupported() { + return in.markSupported(); + } + + @Override + public void mark(int readLimit) { + markTriggeredForCurrentPacket = true; + markWriteBuffer = new ByteArrayOutputStream(); + markPacketIndex = getPacketIndex(); + markReadLimit = readLimit; + } + + @Override + public void reset() throws IOException { + if (markWriteBuffer == null) { + throw new IOException("mark not called"); + } + // mark triggered before the packet boundary has been read over + if (false == markTriggeredForCurrentPacket) { + if (markPacketIndex >= getPacketIndex()) { + throw new IllegalStateException(); + } + in.reset(); + setPacketIndex(markPacketIndex); + } + if (markPacketIndex != getPacketIndex()) { + throw new IllegalStateException(); + } + // make any cached ciphertext available to read + markReadBuffer = new ByteArrayInputStream(markWriteBuffer.toByteArray()); + } + + @Override + void readAtTheStartOfPacketHandler() { + if (markTriggeredForCurrentPacket) { + markTriggeredForCurrentPacket = false; + // mark the underlying stream at the start of the packet + in.mark(markReadLimit); + } + } } private static final int GCM_TAG_SIZE_IN_BYTES = 16; @@ -51,25 +154,19 @@ private GCMPacketsMarkDecorator(InputStream in, SecretKey secretKey, int mode, l private final ByteBuffer packetIV = ByteBuffer.allocate(GCM_IV_SIZE_IN_BYTES); // how much to read from the underlying stream before finishing the current packet and starting the next one private int stillToReadInPacket; - private int packetSizeInBytes; + private final int packetSizeInBytes; - private byte[] inputByteBuffer = new byte[READ_BUFFER_SIZE_IN_BYTES]; - private byte[] processedByteBuffer = new byte[READ_BUFFER_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES]; - private ByteArrayInputStream processedInputStream = new ByteArrayInputStream(new byte[0]); + private final byte[] inputByteBuffer = new byte[READ_BUFFER_SIZE_IN_BYTES]; + private final byte[] processedByteBuffer = new byte[READ_BUFFER_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES]; + private InputStream processedInputStream = InputStream.nullInputStream(); private int bytesBufferedInsideTheCipher = 0; - private ByteArrayOutputStream markWriteBuffer = null; - private ByteArrayInputStream markReadBuffer = new ByteArrayInputStream(new byte[0]); - private boolean markTriggered = false; - private long markPacketIndex; - private int markReadLimit; - static GCMPacketsCipherInputStream getGCMPacketsEncryptor(InputStream in, SecretKey secretKey, int nonce, Provider provider) { - return new GCMPacketsCipherInputStream(in, secretKey, ENCRYPT_MODE, 0, nonce, provider); + return new GCMPacketsWithMarkCipherInputStream(in, secretKey, ENCRYPT_MODE, 0, nonce, provider); } static GCMPacketsCipherInputStream getGCMPacketsDecryptor(InputStream in, SecretKey secretKey, int nonce, Provider provider) { - return new GCMPacketsCipherInputStream(in, secretKey, DECRYPT_MODE, 0, nonce, provider); + return new GCMPacketsWithMarkCipherInputStream(in, secretKey, DECRYPT_MODE, 0, nonce, provider); } public static GCMPacketsCipherInputStream getGCMPacketsEncryptor(InputStream in, SecretKey secretKey, int nonce) { @@ -114,17 +211,20 @@ private GCMPacketsCipherInputStream(InputStream in, SecretKey secretKey, int mod stillToReadInPacket = packetSizeInBytes; } - private void initCipher() throws GeneralSecurityException { + private void reinitPacketCipher() throws GeneralSecurityException { Cipher cipher; if (provider != null) { cipher = Cipher.getInstance(GCM_ENCRYPTION_SCHEME, provider); } else { cipher = Cipher.getInstance(GCM_ENCRYPTION_SCHEME); } + // construct IV and increment packet index packetIV.putLong(0, packetIndex++); GCMParameterSpec gcmParameterSpec = new GCMParameterSpec(GCM_TAG_SIZE_IN_BYTES * Byte.SIZE, packetIV.array()); cipher.init(mode, secretKey, gcmParameterSpec); packetCipher = cipher; + // the new cipher has no bytes buffered inside + bytesBufferedInsideTheCipher = 0; } private int readAndProcess() throws IOException, GeneralSecurityException { @@ -136,8 +236,12 @@ private int readAndProcess() throws IOException, GeneralSecurityException { if (done) { return -1; } + // starting to read a new packet if (stillToReadInPacket == packetSizeInBytes) { - readAtTheStartOfPacket(); + // reinit cipher for this following packet + reinitPacketCipher(); + // call handler to notify subclasses that the processing of a new packet has started + readAtTheStartOfPacketHandler(); } int bytesToRead = Math.min(inputByteBuffer.length - bytesBufferedInsideTheCipher, stillToReadInPacket); if (bytesToRead <= 0) { @@ -200,7 +304,7 @@ private int readAndProcess() throws IOException, GeneralSecurityException { bytesBufferedInsideTheCipher += (bytesRead - bytesProcessed); } } - // the "if" is just an "optimization" + // the "if" here is just an "optimization" if (bytesProcessed != 0) { processedInputStream = new ByteArrayInputStream(processedByteBuffer, 0, bytesProcessed); } @@ -209,10 +313,6 @@ private int readAndProcess() throws IOException, GeneralSecurityException { @Override public int read() throws IOException { - // first try read from the buffered bytes after the mark inside the packet - if (markReadBuffer.available() > 0) { - return markReadBuffer.read(); - } while (processedInputStream.available() <= 0) { try { if (readAndProcess() == -1) { @@ -222,19 +322,11 @@ public int read() throws IOException { throw new IOException(e); } } - int cipherByte = processedInputStream.read(); - if (markTriggered && cipherByte != -1) { - markWriteBuffer.write(cipherByte); - } - return cipherByte; + return processedInputStream.read(); } @Override public int read(byte b[], int off, int len) throws IOException { - int bytesReadFromMarkBuffer = markReadBuffer.read(b, off, len); - if (bytesReadFromMarkBuffer != -1) { - return bytesReadFromMarkBuffer; - } while (processedInputStream.available() <= 0) { try { if (readAndProcess() == -1) { @@ -244,40 +336,17 @@ public int read(byte b[], int off, int len) throws IOException { throw new IOException(e); } } - int bytesReadFromCipherBuffer = processedInputStream.read(b, off, len); - if (markTriggered && bytesReadFromCipherBuffer != -1) { - markWriteBuffer.write(b, off, bytesReadFromCipherBuffer); - } - return bytesReadFromCipherBuffer; + return processedInputStream.read(b, off, len); } @Override public long skip(long n) throws IOException { - if (markReadBuffer.available() > 0) { - return markReadBuffer.skip(n); - } - // if mark is triggered bytes cannot be discarded because they might be read later - if (markTriggered) { - // mark - processedInputStream.mark(packetSizeInBytes); - // skip - int skipAheadBytes = Math.toIntExact(processedInputStream.skip(n)); - // reset - processedInputStream.reset(); - byte[] temp = new byte[skipAheadBytes]; - // re-read the skipped bytes - processedInputStream.read(temp); - // do not discard the skipped bytes - markWriteBuffer.write(temp); - return skipAheadBytes; - } else { - return processedInputStream.skip(n); - } + return processedInputStream.skip(n); } @Override public int available() throws IOException { - return markReadBuffer.available() + processedInputStream.available(); + return processedInputStream.available(); } @Override @@ -286,7 +355,7 @@ public void close() throws IOException { return; } closed = true; - processedInputStream = new ByteArrayInputStream(new byte[0]); + processedInputStream = InputStream.nullInputStream(); in.close(); // Throw away the unprocessed data and throw no crypto exceptions. // Normally the GCM cipher is fully readed before closing, so any authentication @@ -303,37 +372,37 @@ public void close() throws IOException { @Override public boolean markSupported() { - return in.markSupported(); + return false; } @Override public void mark(int readLimit) { - markTriggered = true; - markWriteBuffer = new ByteArrayOutputStream(); - markPacketIndex = packetIndex; - markReadLimit = readLimit; } @Override public void reset() throws IOException { - if (markWriteBuffer == null) { - throw new IOException(); - } - if (false == markTriggered) { - in.reset(); - processedInputStream = new ByteArrayInputStream(new byte[0]); - } - markReadBuffer = new ByteArrayInputStream(markWriteBuffer.toByteArray()); - packetIndex = markPacketIndex; + throw new IOException("mark/reset not supported"); + } + + /** + * Sets the packet index and clears the transitory state from processing of the previous packet + */ + void setPacketIndex(long packetIndex) { + processedInputStream = InputStream.nullInputStream(); + stillToReadInPacket = packetSizeInBytes; + this.packetIndex = packetIndex; + } + + long getPacketIndex() { + return packetIndex; } - private void readAtTheStartOfPacket() throws GeneralSecurityException { - // re init cipher for this following packet - initCipher(); - if (markTriggered) { - markTriggered = false; - // mark the underlying stream at the start of the packet - in.mark(markReadLimit); + void readAtTheStartOfPacketHandler() { + } + + void ensureOpen() throws IOException { + if (closed) { + throw new IOException("Stream closed"); } } } From c8faeac9b3cd6e48c0c21e781985c5359ed15ad7 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Wed, 16 Oct 2019 17:57:31 +0300 Subject: [PATCH 10/29] Refactored tests to use on large global array --- .../GCMPacketsCipherInputStreamTests.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java index 7c96ff57b39e6..4de71d9e89434 100644 --- a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java +++ b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java @@ -15,10 +15,13 @@ import java.security.AccessController; import java.security.PrivilegedAction; import java.security.SecureRandom; +import java.util.Arrays; import java.util.Random; public class GCMPacketsCipherInputStreamTests extends ESTestCase { + private static int TEST_ARRAY_SIZE = 5 * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES; + private static byte[] testPlaintextArray; private static BouncyCastleFipsProvider bcFipsProvider; private SecretKey secretKey; @@ -26,11 +29,10 @@ public class GCMPacketsCipherInputStreamTests extends ESTestCase { static void setupProvider() { AccessController.doPrivileged((PrivilegedAction) () -> { GCMPacketsCipherInputStreamTests.bcFipsProvider = new BouncyCastleFipsProvider(); -// for (Object o : this.bcFipsProvider.keySet()) { -// System.out.println(o); -// } return null; }); + testPlaintextArray = new byte[TEST_ARRAY_SIZE]; + new Random().nextBytes(testPlaintextArray); } @Before @@ -67,21 +69,21 @@ public void testEncryptDecryptLargerThanPacketSize() throws Exception { } public void testEncryptDecryptMultipleOfPacketSize() throws Exception { - for (int i = 1; i < 10; i++) { + for (int i = 1; i <= 5; i++) { testEncryptDecryptRandomOfLength(i * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES, secretKey); } } private void testEncryptDecryptRandomOfLength(int length, SecretKey secretKey) throws Exception { Random random = new Random(); - byte[] plaintextArray = new byte[length]; - random.nextBytes(plaintextArray); int nonce = random.nextInt(); ByteArrayOutputStream cipherTextOutput; ByteArrayOutputStream plainTextOutput; + int startIndex = randomIntBetween(0, testPlaintextArray.length - length); // encrypt - try (InputStream cipherInputStream = GCMPacketsCipherInputStream.getGCMPacketsEncryptor(new ByteArrayInputStream(plaintextArray), - secretKey, nonce, bcFipsProvider)) { + try (InputStream cipherInputStream = + GCMPacketsCipherInputStream.getGCMPacketsEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), + secretKey, nonce, bcFipsProvider)) { cipherTextOutput = readAllInputStream(cipherInputStream, GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length)); } //decrypt @@ -91,7 +93,7 @@ private void testEncryptDecryptRandomOfLength(int length, SecretKey secretKey) t plainTextOutput = readAllInputStream(plainInputStream, GCMPacketsCipherInputStream.getDecryptionSizeFromCipherSize(cipherTextOutput.size())); } - assertThat(plainTextOutput.toByteArray(), Matchers.is(plaintextArray)); + assertTrue(Arrays.equals(plainTextOutput.toByteArray(), 0, length, testPlaintextArray, startIndex, startIndex + length)); } private SecretKey generateSecretKey() throws Exception { @@ -106,6 +108,7 @@ private SecretKey generateSecretKey() throws Exception { }); } + // read "adversarily" in small random pieces private ByteArrayOutputStream readAllInputStream(InputStream inputStream, int size) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(size); byte[] temp = new byte[randomIntBetween(1, size != 0 ? size : 1)]; From 4052dcdabcf0426945cd1f7ed72647aded02e200 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 17 Oct 2019 13:26:24 +0300 Subject: [PATCH 11/29] Fix mark reset bug --- .../GCMPacketsCipherInputStream.java | 32 +++-- .../GCMPacketsCipherInputStreamTests.java | 109 +++++++++++++++++- 2 files changed, 125 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java index 77b5ca2713440..b16949237fc40 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java @@ -63,7 +63,7 @@ public int read(byte b[], int off, int len) throws IOException { } int cipherBytesCount = super.read(b, off, len); // if buffering of the ciphertext is required - if (markTriggeredForCurrentPacket && cipherBytesCount != -1) { + if (markTriggeredForCurrentPacket && cipherBytesCount > 0) { markWriteBuffer.write(b, off, cipherBytesCount ); } return cipherBytesCount; @@ -102,6 +102,15 @@ public boolean markSupported() { public void mark(int readLimit) { markTriggeredForCurrentPacket = true; markWriteBuffer = new ByteArrayOutputStream(); + if (markReadBuffer.available() > 0) { + markReadBuffer.mark(Integer.MAX_VALUE); + try { + markReadBuffer.transferTo(markWriteBuffer); + } catch (IOException e) { + throw new IllegalStateException(e); + } + markReadBuffer.reset(); + } markPacketIndex = getPacketIndex(); markReadLimit = readLimit; } @@ -111,12 +120,14 @@ public void reset() throws IOException { if (markWriteBuffer == null) { throw new IOException("mark not called"); } + if (markPacketIndex > getPacketIndex()) { + throw new IllegalStateException(); + } // mark triggered before the packet boundary has been read over if (false == markTriggeredForCurrentPacket) { - if (markPacketIndex >= getPacketIndex()) { - throw new IllegalStateException(); - } + // reset underlying input stream to packet boundary in.reset(); + // set packet index for the next packet and clear any transitory state of any inside of packet processing setPacketIndex(markPacketIndex); } if (markPacketIndex != getPacketIndex()) { @@ -161,20 +172,20 @@ void readAtTheStartOfPacketHandler() { private InputStream processedInputStream = InputStream.nullInputStream(); private int bytesBufferedInsideTheCipher = 0; - static GCMPacketsCipherInputStream getGCMPacketsEncryptor(InputStream in, SecretKey secretKey, int nonce, Provider provider) { + static GCMPacketsCipherInputStream getEncryptor(InputStream in, SecretKey secretKey, int nonce, Provider provider) { return new GCMPacketsWithMarkCipherInputStream(in, secretKey, ENCRYPT_MODE, 0, nonce, provider); } - static GCMPacketsCipherInputStream getGCMPacketsDecryptor(InputStream in, SecretKey secretKey, int nonce, Provider provider) { + static GCMPacketsCipherInputStream getDecryptor(InputStream in, SecretKey secretKey, int nonce, Provider provider) { return new GCMPacketsWithMarkCipherInputStream(in, secretKey, DECRYPT_MODE, 0, nonce, provider); } - public static GCMPacketsCipherInputStream getGCMPacketsEncryptor(InputStream in, SecretKey secretKey, int nonce) { - return getGCMPacketsEncryptor(in, secretKey, nonce, new BouncyCastleFipsProvider()); + public static GCMPacketsCipherInputStream getEncryptor(InputStream in, SecretKey secretKey, int nonce) { + return getEncryptor(in, secretKey, nonce, new BouncyCastleFipsProvider()); } - public static GCMPacketsCipherInputStream getGCMPacketsDecryptor(InputStream in, SecretKey secretKey, int nonce) { - return getGCMPacketsDecryptor(in, secretKey, nonce, new BouncyCastleFipsProvider()); + public static GCMPacketsCipherInputStream getDecryptor(InputStream in, SecretKey secretKey, int nonce) { + return getDecryptor(in, secretKey, nonce, new BouncyCastleFipsProvider()); } public static int getEncryptionSizeFromPlainSize(int size) { @@ -390,6 +401,7 @@ public void reset() throws IOException { void setPacketIndex(long packetIndex) { processedInputStream = InputStream.nullInputStream(); stillToReadInPacket = packetSizeInBytes; + done = false; this.packetIndex = packetIndex; } diff --git a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java index 4de71d9e89434..8f741496a74fa 100644 --- a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java +++ b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java @@ -20,7 +20,8 @@ public class GCMPacketsCipherInputStreamTests extends ESTestCase { - private static int TEST_ARRAY_SIZE = 5 * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES; + private static int TEST_ARRAY_SIZE = 8 * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES; + private static int ENCRYPTED_PACKET_SIZE = GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES); private static byte[] testPlaintextArray; private static BouncyCastleFipsProvider bcFipsProvider; private SecretKey secretKey; @@ -63,32 +64,128 @@ public void testEncryptDecryptSmallerThanPacketSize() throws Exception { } public void testEncryptDecryptLargerThanPacketSize() throws Exception { - for (int i = GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES + 1; i < GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES * 3; i++) { + for (int i = GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES + 1; i <= GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES * 4; i++) { testEncryptDecryptRandomOfLength(i, secretKey); } } public void testEncryptDecryptMultipleOfPacketSize() throws Exception { - for (int i = 1; i <= 5; i++) { + for (int i = 1; i <= 7; i++) { testEncryptDecryptRandomOfLength(i * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES, secretKey); } } + public void testMarkAndResetAtBeginningForEncryption() throws Exception { + testMarkAndResetToSameOffsetForEncryption(0); + testMarkAndResetToSameOffsetForEncryption(GCMPacketsCipherInputStream. + getEncryptionSizeFromPlainSize(GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES)); + } + + public void testMarkAndResetFirstPacketForEncryption() throws Exception { + for (int i = 1; i < ENCRYPTED_PACKET_SIZE; i++) { + testMarkAndResetToSameOffsetForEncryption(i); + } + } + + public void testMarkAndResetRandomSecondPacketForEncryption() throws Exception { + for (int i = ENCRYPTED_PACKET_SIZE + 1; i < 2 * ENCRYPTED_PACKET_SIZE; i++) { + testMarkAndResetToSameOffsetForEncryption(i); + } + } + + public void testMarkAndResetCrawlForEncryption() throws Exception { + int length = GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES + randomIntBetween(0, + GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES); + int startIndex = randomIntBetween(0, testPlaintextArray.length - length); + int nonce = new Random().nextInt(); + byte[] ciphertextBytes; + try (InputStream cipherInputStream = + GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), + secretKey, nonce, bcFipsProvider)) { + ciphertextBytes = cipherInputStream.readAllBytes(); + } + try (InputStream cipherInputStream = + GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), + secretKey, nonce, bcFipsProvider)) { + cipherInputStream.mark(Integer.MAX_VALUE); + for (int i = 0; i < GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length); i++) { + int skipSize = randomIntBetween(1, GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length) - i); + cipherInputStream.readNBytes(skipSize); + cipherInputStream.reset(); + int byteRead = cipherInputStream.read(); + cipherInputStream.mark(Integer.MAX_VALUE); + assertThat("Mismatch at position: " + i, (byte) byteRead, Matchers.is(ciphertextBytes[i])); + } + } + } + + private void testMarkAndResetToSameOffsetForEncryption(int offset) throws Exception { + int length = 3 * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES + randomIntBetween(0, + GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES); + int startIndex = randomIntBetween(0, testPlaintextArray.length - length); + try (InputStream cipherInputStream = + GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), + secretKey, new Random().nextInt(), bcFipsProvider)) { + // skip offset bytes + cipherInputStream.readNBytes(offset); + // mark after offset + cipherInputStream.mark(Integer.MAX_VALUE); + // read/skip less than (encrypted) packet size + int skipSize = randomIntBetween(1, ENCRYPTED_PACKET_SIZE - 1); + byte[] firstPassEncryption = cipherInputStream.readNBytes(skipSize); + // back to start + cipherInputStream.reset(); + // read/skip more than (encrypted) packet size, but less than the full stream + skipSize = randomIntBetween(ENCRYPTED_PACKET_SIZE, + GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length) - 1 - offset); + byte[] secondPassEncryption = cipherInputStream.readNBytes(skipSize); + assertTrue(Arrays.equals(firstPassEncryption, 0, firstPassEncryption.length, secondPassEncryption, 0, + firstPassEncryption.length)); + // back to start + cipherInputStream.reset(); + byte[] thirdPassEncryption; + // read/skip to end of ciphertext + if (randomBoolean()) { + thirdPassEncryption = cipherInputStream.readAllBytes(); + } else { + thirdPassEncryption = + cipherInputStream.readNBytes(GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length) - offset); + } + assertTrue(Arrays.equals(secondPassEncryption, 0, secondPassEncryption.length, thirdPassEncryption, 0, + secondPassEncryption.length)); + // back to start + cipherInputStream.reset(); + // read/skip more than (encrypted) packet size, but less than the full stream + skipSize = randomIntBetween(ENCRYPTED_PACKET_SIZE, + GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length) - 1 - offset); + byte[] fourthPassEncryption = cipherInputStream.readNBytes(skipSize); + assertTrue(Arrays.equals(fourthPassEncryption, 0, fourthPassEncryption.length, thirdPassEncryption, 0, + fourthPassEncryption.length)); + // back to start + cipherInputStream.reset(); + // read/skip less than (encrypted) packet size + skipSize = randomIntBetween(1, ENCRYPTED_PACKET_SIZE - 1); + byte[] fifthsPassEncryption = cipherInputStream.readNBytes(skipSize); + assertTrue(Arrays.equals(fifthsPassEncryption, 0, fifthsPassEncryption.length, fourthPassEncryption, 0, + fifthsPassEncryption.length)); + } + } + private void testEncryptDecryptRandomOfLength(int length, SecretKey secretKey) throws Exception { Random random = new Random(); int nonce = random.nextInt(); ByteArrayOutputStream cipherTextOutput; ByteArrayOutputStream plainTextOutput; - int startIndex = randomIntBetween(0, testPlaintextArray.length - length); + int startIndex = randomIntBetween(0, testPlaintextArray.length - length); // encrypt try (InputStream cipherInputStream = - GCMPacketsCipherInputStream.getGCMPacketsEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), + GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), secretKey, nonce, bcFipsProvider)) { cipherTextOutput = readAllInputStream(cipherInputStream, GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length)); } //decrypt try (InputStream plainInputStream = - GCMPacketsCipherInputStream.getGCMPacketsDecryptor(new ByteArrayInputStream(cipherTextOutput.toByteArray()), + GCMPacketsCipherInputStream.getDecryptor(new ByteArrayInputStream(cipherTextOutput.toByteArray()), secretKey, nonce, bcFipsProvider)) { plainTextOutput = readAllInputStream(plainInputStream, GCMPacketsCipherInputStream.getDecryptionSizeFromCipherSize(cipherTextOutput.size())); From 33133fcf4fa5c468d50f362767ded9ccadf07c46 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 17 Oct 2019 13:52:39 +0300 Subject: [PATCH 12/29] Tests counters --- .../encrypted/GCMPacketsCipherInputStreamTests.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java index 8f741496a74fa..32b84f43336fc 100644 --- a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java +++ b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java @@ -64,13 +64,13 @@ public void testEncryptDecryptSmallerThanPacketSize() throws Exception { } public void testEncryptDecryptLargerThanPacketSize() throws Exception { - for (int i = GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES + 1; i <= GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES * 4; i++) { + for (int i = GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES + 1; i <= GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES * 3; i++) { testEncryptDecryptRandomOfLength(i, secretKey); } } public void testEncryptDecryptMultipleOfPacketSize() throws Exception { - for (int i = 1; i <= 7; i++) { + for (int i = 1; i <= 6; i++) { testEncryptDecryptRandomOfLength(i * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES, secretKey); } } @@ -94,8 +94,7 @@ public void testMarkAndResetRandomSecondPacketForEncryption() throws Exception { } public void testMarkAndResetCrawlForEncryption() throws Exception { - int length = GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES + randomIntBetween(0, - GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES); + int length = GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES; int startIndex = randomIntBetween(0, testPlaintextArray.length - length); int nonce = new Random().nextInt(); byte[] ciphertextBytes; @@ -120,7 +119,7 @@ public void testMarkAndResetCrawlForEncryption() throws Exception { } private void testMarkAndResetToSameOffsetForEncryption(int offset) throws Exception { - int length = 3 * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES + randomIntBetween(0, + int length = 4 * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES + randomIntBetween(0, GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES); int startIndex = randomIntBetween(0, testPlaintextArray.length - length); try (InputStream cipherInputStream = From b25598a0f67160c6937db90ac9167296378e5a3e Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 17 Oct 2019 17:58:14 +0300 Subject: [PATCH 13/29] Reset in rewind tests --- .../GCMPacketsCipherInputStream.java | 11 +- .../GCMPacketsCipherInputStreamTests.java | 107 ++++++++++++++++-- 2 files changed, 107 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java index b16949237fc40..d3a57dff87421 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java @@ -152,6 +152,7 @@ void readAtTheStartOfPacketHandler() { private static final String GCM_ENCRYPTION_SCHEME = "AES/GCM/NoPadding"; static final int PACKET_SIZE_IN_BYTES = 4096; + static final int ENCRYPTED_PACKET_SIZE_IN_BYTES = PACKET_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES; static final int READ_BUFFER_SIZE_IN_BYTES = 512; private boolean done = false; @@ -192,16 +193,18 @@ public static int getEncryptionSizeFromPlainSize(int size) { if (size < 0) { throw new IllegalArgumentException(); } - return (size / PACKET_SIZE_IN_BYTES) * (PACKET_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES) - + (size % PACKET_SIZE_IN_BYTES) + GCM_TAG_SIZE_IN_BYTES; + return (size / PACKET_SIZE_IN_BYTES) * (ENCRYPTED_PACKET_SIZE_IN_BYTES) + (size % PACKET_SIZE_IN_BYTES) + GCM_TAG_SIZE_IN_BYTES; } public static int getDecryptionSizeFromCipherSize(int size) { if (size < GCM_TAG_SIZE_IN_BYTES) { throw new IllegalArgumentException(); } - return (size / (PACKET_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES)) * PACKET_SIZE_IN_BYTES - + (size % (PACKET_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES)) - GCM_TAG_SIZE_IN_BYTES; + int plainSize = (size / (ENCRYPTED_PACKET_SIZE_IN_BYTES)) * PACKET_SIZE_IN_BYTES; + if (size % ENCRYPTED_PACKET_SIZE_IN_BYTES < GCM_TAG_SIZE_IN_BYTES) { + throw new IllegalArgumentException(); + } + return plainSize + (size % ENCRYPTED_PACKET_SIZE_IN_BYTES) - GCM_TAG_SIZE_IN_BYTES; } private GCMPacketsCipherInputStream(InputStream in, SecretKey secretKey, int mode, long packetIndex, int nonce, Provider provider) { diff --git a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java index 32b84f43336fc..5c68c3c24997d 100644 --- a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java +++ b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java @@ -18,10 +18,11 @@ import java.util.Arrays; import java.util.Random; +import static org.elasticsearch.repositories.encrypted.GCMPacketsCipherInputStream.ENCRYPTED_PACKET_SIZE_IN_BYTES; + public class GCMPacketsCipherInputStreamTests extends ESTestCase { private static int TEST_ARRAY_SIZE = 8 * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES; - private static int ENCRYPTED_PACKET_SIZE = GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES); private static byte[] testPlaintextArray; private static BouncyCastleFipsProvider bcFipsProvider; private SecretKey secretKey; @@ -82,13 +83,13 @@ public void testMarkAndResetAtBeginningForEncryption() throws Exception { } public void testMarkAndResetFirstPacketForEncryption() throws Exception { - for (int i = 1; i < ENCRYPTED_PACKET_SIZE; i++) { + for (int i = 1; i < ENCRYPTED_PACKET_SIZE_IN_BYTES; i++) { testMarkAndResetToSameOffsetForEncryption(i); } } public void testMarkAndResetRandomSecondPacketForEncryption() throws Exception { - for (int i = ENCRYPTED_PACKET_SIZE + 1; i < 2 * ENCRYPTED_PACKET_SIZE; i++) { + for (int i = ENCRYPTED_PACKET_SIZE_IN_BYTES + 1; i < 2 * ENCRYPTED_PACKET_SIZE_IN_BYTES; i++) { testMarkAndResetToSameOffsetForEncryption(i); } } @@ -109,15 +110,107 @@ public void testMarkAndResetCrawlForEncryption() throws Exception { cipherInputStream.mark(Integer.MAX_VALUE); for (int i = 0; i < GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length); i++) { int skipSize = randomIntBetween(1, GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length) - i); + // skip bytes cipherInputStream.readNBytes(skipSize); cipherInputStream.reset(); + // re-read one byte of the skipped bytes int byteRead = cipherInputStream.read(); + // mark the one byte progress cipherInputStream.mark(Integer.MAX_VALUE); assertThat("Mismatch at position: " + i, (byte) byteRead, Matchers.is(ciphertextBytes[i])); } } } + public void testMarkAndResetStepInRewindBuffer() throws Exception { + int length = 2 * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES; + int startIndex = randomIntBetween(0, testPlaintextArray.length - length); + int nonce = new Random().nextInt(); + byte[] ciphertextBytes; + try (InputStream cipherInputStream = + GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), + secretKey, nonce, bcFipsProvider)) { + ciphertextBytes = cipherInputStream.readAllBytes(); + } + try (InputStream cipherInputStream = + GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), + secretKey, nonce, bcFipsProvider)) { + int position1 = randomIntBetween(1, ENCRYPTED_PACKET_SIZE_IN_BYTES - 2); + int position2 = randomIntBetween(position1 + 1, ENCRYPTED_PACKET_SIZE_IN_BYTES - 1); + int position3 = ENCRYPTED_PACKET_SIZE_IN_BYTES; + int position4 = randomIntBetween(position3 + 1, 2 * ENCRYPTED_PACKET_SIZE_IN_BYTES - 2); + int position5 = randomIntBetween(position4 + 1, 2 * ENCRYPTED_PACKET_SIZE_IN_BYTES - 1); + int position6 = 2 * ENCRYPTED_PACKET_SIZE_IN_BYTES; + int position7 = GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length); + // skip position1 bytes + cipherInputStream.readNBytes(position1); + // mark position1 + cipherInputStream.mark(Integer.MAX_VALUE); + byte[] bytesPos17; + if (randomBoolean()) { + bytesPos17 = cipherInputStream.readAllBytes(); + } else { + bytesPos17 = cipherInputStream.readNBytes(position7 - position1); + } + // reset back to position 1 + cipherInputStream.reset(); + byte[] bytesPos12 = cipherInputStream.readNBytes(position2 - position1); + assertTrue(Arrays.equals(bytesPos12, 0, bytesPos12.length, bytesPos17, 0, bytesPos12.length)); + // mark position2 + cipherInputStream.mark(Integer.MAX_VALUE); + byte[] bytesPos26 = cipherInputStream.readNBytes(position6 - position2); + assertTrue(Arrays.equals(bytesPos26, 0, bytesPos26.length, bytesPos17, (position2 - position1), + (position2 - position1) + bytesPos26.length)); + // reset to position 2 + cipherInputStream.reset(); + byte[] bytesPos23 = cipherInputStream.readNBytes(position3 - position2); + assertTrue(Arrays.equals(bytesPos23, 0, bytesPos23.length, bytesPos17, (position2 - position1), + (position2 - position1) + bytesPos23.length)); + // mark position3 + cipherInputStream.mark(Integer.MAX_VALUE); + byte[] bytesPos36 = cipherInputStream.readNBytes(position6 - position3); + assertTrue(Arrays.equals(bytesPos36, 0, bytesPos36.length, bytesPos17, (position3 - position1), + (position3 - position1) + bytesPos36.length)); + // reset to position 3 + cipherInputStream.reset(); + byte[] bytesPos34 = cipherInputStream.readNBytes(position4 - position3); + assertTrue(Arrays.equals(bytesPos34, 0, bytesPos34.length, bytesPos17, (position3 - position1), + (position3 - position1) + bytesPos34.length)); + // mark position4 + cipherInputStream.mark(Integer.MAX_VALUE); + byte[] bytesPos46 = cipherInputStream.readNBytes(position6 - position4); + assertTrue(Arrays.equals(bytesPos46, 0, bytesPos46.length, bytesPos17, (position4 - position1), + (position4 - position1) + bytesPos46.length)); + // reset to position 4 + cipherInputStream.reset(); + byte[] bytesPos45 = cipherInputStream.readNBytes(position5 - position4); + assertTrue(Arrays.equals(bytesPos45, 0, bytesPos45.length, bytesPos17, (position4 - position1), + (position4 - position1) + bytesPos45.length)); + // mark position 5 + cipherInputStream.mark(Integer.MAX_VALUE); + byte[] bytesPos56 = cipherInputStream.readNBytes(position6 - position5); + assertTrue(Arrays.equals(bytesPos56, 0, bytesPos56.length, bytesPos17, (position5 - position1), + (position5 - position1) + bytesPos56.length)); + // mark position 6 + cipherInputStream.mark(Integer.MAX_VALUE); + byte[] bytesPos67; + if (randomBoolean()) { + bytesPos67 = cipherInputStream.readAllBytes(); + } else { + bytesPos67 = cipherInputStream.readNBytes(position7 - position6); + } + assertTrue(Arrays.equals(bytesPos67, 0, bytesPos67.length, bytesPos17, (position6 - position1), + (position6 - position1) + bytesPos67.length)); + // mark position 7 (end of stream) + cipherInputStream.mark(Integer.MAX_VALUE); + // end of stream + assertThat(cipherInputStream.read(), Matchers.is(-1)); + // reset at the end + cipherInputStream.reset(); + assertThat(cipherInputStream.read(), Matchers.is(-1)); + } + } + private void testMarkAndResetToSameOffsetForEncryption(int offset) throws Exception { int length = 4 * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES + randomIntBetween(0, GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES); @@ -130,12 +223,12 @@ secretKey, new Random().nextInt(), bcFipsProvider)) { // mark after offset cipherInputStream.mark(Integer.MAX_VALUE); // read/skip less than (encrypted) packet size - int skipSize = randomIntBetween(1, ENCRYPTED_PACKET_SIZE - 1); + int skipSize = randomIntBetween(1, ENCRYPTED_PACKET_SIZE_IN_BYTES - 1); byte[] firstPassEncryption = cipherInputStream.readNBytes(skipSize); // back to start cipherInputStream.reset(); // read/skip more than (encrypted) packet size, but less than the full stream - skipSize = randomIntBetween(ENCRYPTED_PACKET_SIZE, + skipSize = randomIntBetween(ENCRYPTED_PACKET_SIZE_IN_BYTES, GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length) - 1 - offset); byte[] secondPassEncryption = cipherInputStream.readNBytes(skipSize); assertTrue(Arrays.equals(firstPassEncryption, 0, firstPassEncryption.length, secondPassEncryption, 0, @@ -155,7 +248,7 @@ secretKey, new Random().nextInt(), bcFipsProvider)) { // back to start cipherInputStream.reset(); // read/skip more than (encrypted) packet size, but less than the full stream - skipSize = randomIntBetween(ENCRYPTED_PACKET_SIZE, + skipSize = randomIntBetween(ENCRYPTED_PACKET_SIZE_IN_BYTES, GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length) - 1 - offset); byte[] fourthPassEncryption = cipherInputStream.readNBytes(skipSize); assertTrue(Arrays.equals(fourthPassEncryption, 0, fourthPassEncryption.length, thirdPassEncryption, 0, @@ -163,7 +256,7 @@ secretKey, new Random().nextInt(), bcFipsProvider)) { // back to start cipherInputStream.reset(); // read/skip less than (encrypted) packet size - skipSize = randomIntBetween(1, ENCRYPTED_PACKET_SIZE - 1); + skipSize = randomIntBetween(1, ENCRYPTED_PACKET_SIZE_IN_BYTES - 1); byte[] fifthsPassEncryption = cipherInputStream.readNBytes(skipSize); assertTrue(Arrays.equals(fifthsPassEncryption, 0, fifthsPassEncryption.length, fourthPassEncryption, 0, fifthsPassEncryption.length)); From 710ea8a124f16a921078bd58c39589ba0ba3afe4 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 17 Oct 2019 18:47:27 +0300 Subject: [PATCH 14/29] Failed decryption tests --- .../GCMPacketsCipherInputStreamTests.java | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java index 5c68c3c24997d..bb8bf6ffd980d 100644 --- a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java +++ b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java @@ -6,6 +6,7 @@ import org.junit.Before; import org.junit.BeforeClass; +import javax.crypto.AEADBadTagException; import javax.crypto.KeyGenerator; import javax.crypto.SecretKey; import java.io.ByteArrayInputStream; @@ -19,6 +20,7 @@ import java.util.Random; import static org.elasticsearch.repositories.encrypted.GCMPacketsCipherInputStream.ENCRYPTED_PACKET_SIZE_IN_BYTES; +import static org.elasticsearch.repositories.encrypted.GCMPacketsCipherInputStream.READ_BUFFER_SIZE_IN_BYTES; public class GCMPacketsCipherInputStreamTests extends ESTestCase { @@ -211,6 +213,67 @@ public void testMarkAndResetStepInRewindBuffer() throws Exception { } } + public void testDecryptionFails() throws Exception { + Random random = new Random(); + int length = randomIntBetween(0, READ_BUFFER_SIZE_IN_BYTES); + int startIndex = randomIntBetween(0, testPlaintextArray.length - length); + int nonce = new Random().nextInt(); + byte[] ciphertextBytes; + try (InputStream cipherInputStream = + GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), + secretKey, nonce, bcFipsProvider)) { + ciphertextBytes = cipherInputStream.readAllBytes(); + } + // decryption fails for one byte modifications + for (int i = 0; i < ciphertextBytes.length; i++) { + byte bytei = ciphertextBytes[i]; + while (bytei == ciphertextBytes[i]) { + ciphertextBytes[i] = (byte) random.nextInt(); + } + try (InputStream plainInputStream = + GCMPacketsCipherInputStream.getDecryptor(new ByteArrayInputStream(ciphertextBytes), + secretKey, nonce, bcFipsProvider)) { + IOException e = expectThrows(IOException.class, () -> { + readAllInputStream(plainInputStream, + GCMPacketsCipherInputStream.getDecryptionSizeFromCipherSize(ciphertextBytes.length)); + }); + assertThat(e.getCause(), Matchers.isA(AEADBadTagException.class)); + } + ciphertextBytes[i] = bytei; + } + // decryption fails for one byte omissions + byte[] missingByteCiphertext = new byte[ciphertextBytes.length - 1]; + for (int i = 0; i < ciphertextBytes.length; i++) { + System.arraycopy(ciphertextBytes, 0, missingByteCiphertext, 0, i); + System.arraycopy(ciphertextBytes, i + 1, missingByteCiphertext, i, (ciphertextBytes.length - i - 1)); + try (InputStream plainInputStream = + GCMPacketsCipherInputStream.getDecryptor(new ByteArrayInputStream(missingByteCiphertext), + secretKey, nonce, bcFipsProvider)) { + IOException e = expectThrows(IOException.class, () -> { + readAllInputStream(plainInputStream, + GCMPacketsCipherInputStream.getDecryptionSizeFromCipherSize(missingByteCiphertext.length)); + }); + assertThat(e.getCause(), Matchers.isA(AEADBadTagException.class)); + } + } + // decryption fails for one extra byte + byte[] extraByteCiphertext = new byte[ciphertextBytes.length + 1]; + for (int i = 0; i < ciphertextBytes.length; i++) { + System.arraycopy(ciphertextBytes, 0, extraByteCiphertext, 0, i); + extraByteCiphertext[i] = (byte) random.nextInt(); + System.arraycopy(ciphertextBytes, i, extraByteCiphertext, i + 1, (ciphertextBytes.length - i)); + try (InputStream plainInputStream = + GCMPacketsCipherInputStream.getDecryptor(new ByteArrayInputStream(extraByteCiphertext), + secretKey, nonce, bcFipsProvider)) { + IOException e = expectThrows(IOException.class, () -> { + readAllInputStream(plainInputStream, + GCMPacketsCipherInputStream.getDecryptionSizeFromCipherSize(extraByteCiphertext.length)); + }); + assertThat(e.getCause(), Matchers.isA(AEADBadTagException.class)); + } + } + } + private void testMarkAndResetToSameOffsetForEncryption(int offset) throws Exception { int length = 4 * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES + randomIntBetween(0, GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES); From 1a7801f95976c5753fb85f64205be6d6ff404175 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Fri, 18 Oct 2019 00:04:45 +0300 Subject: [PATCH 15/29] Adapted Encrypted Repository but it does not work yet --- .../blobstore/BlobStoreRepository.java | 2 +- .../encrypted/EncryptedRepository.java | 91 ++++++++----------- .../GCMPacketsCipherInputStream.java | 13 +-- .../GCMPacketsCipherInputStreamTests.java | 22 ++--- 4 files changed, 58 insertions(+), 70 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 162932477a4d7..a244d97801e69 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -350,7 +350,7 @@ protected final boolean isCompress() { * * @return chunk size */ - public ByteSizeValue chunkSize() { + protected ByteSizeValue chunkSize() { return null; } diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java index db0b719014c14..6b561215eec85 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java @@ -14,13 +14,13 @@ import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.blobstore.DeleteResult; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.hash.MessageDigests; +import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.SecureSetting; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeUnit; -import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; @@ -30,13 +30,14 @@ import javax.crypto.NoSuchPaddingException; import javax.crypto.SecretKey; import javax.crypto.SecretKeyFactory; -import javax.crypto.spec.GCMParameterSpec; import javax.crypto.spec.PBEKeySpec; import javax.crypto.spec.SecretKeySpec; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; import java.security.spec.InvalidKeySpecException; import java.util.HashMap; import java.util.Map; @@ -44,19 +45,15 @@ public class EncryptedRepository extends BlobStoreRepository { + private static final BouncyCastleFipsProvider BC_FIPS_PROVIDER = new BouncyCastleFipsProvider(); + // TODO change this and make it (secure) randomly generated for each blob + private static final int NONCE = 5318008; + static final Setting.AffixSetting ENCRYPTION_PASSWORD_SETTING = Setting.affixKeySetting("repository.encrypted.", "password", key -> SecureSetting.secureString(key, null)); private static final Setting DELEGATE_TYPE = new Setting<>("delegate_type", "", Function.identity()); - private static final int GCM_TAG_BYTES_LENGTH = 16; - private static final String ENCRYPTION_MODE = "AES/GCM/NoPadding"; private static final String ENCRYPTION_METADATA_PREFIX = "encryption-metadata-"; - // always the same IV because the key is randomly generated anew (Key-IV pair is never repeated) - private static final GCMParameterSpec gcmParameterSpec = new GCMParameterSpec(128, new byte[] {0,1,2,3,4,5,6,7,8,9,10,11 }); - // given the mode, the IV and the tag length, the maximum "chunk" size is ~64GB, we set it to 32GB to err on the safe side - public static final ByteSizeValue MAX_CHUNK_SIZE = new ByteSizeValue(32, ByteSizeUnit.GB); - - private static final BouncyCastleFipsProvider BC_FIPS_PROV = new BouncyCastleFipsProvider(); private final BlobStoreRepository delegatedRepository; private final SecretKey masterSecretKey; @@ -90,16 +87,6 @@ protected void doClose() { this.delegatedRepository.close(); } - @Override - public ByteSizeValue chunkSize() { - ByteSizeValue delegatedChunkSize = this.delegatedRepository.chunkSize(); - if (delegatedChunkSize == null || delegatedChunkSize.compareTo(MAX_CHUNK_SIZE) > 0) { - return MAX_CHUNK_SIZE; - } else { - return delegatedChunkSize; - } - } - /** * Returns a new encrypted repository factory */ @@ -188,35 +175,35 @@ public BlobPath path() { @Override public InputStream readBlob(String blobName) throws IOException { -// final BytesReference dataDecryptionKeyBytes = Streams.readFully(this.encryptionMetadataBlobContainer.readBlob(blobName)); -// try { -// SecretKey dataDecryptionKey = unwrapKey(BytesReference.toBytes(dataDecryptionKeyBytes), this.masterSecretKey); -// Cipher cipher = Cipher.getInstance(ENCRYPTION_MODE, BC_FIPS_PROV); -// cipher.init(Cipher.DECRYPT_MODE, dataDecryptionKey, ivParameterSpec); -// return new CipherInputStream(this.delegatedBlobContainer.readBlob(blobName), cipher); -// } catch (InvalidKeyException | NoSuchAlgorithmException | NoSuchPaddingException | InvalidAlgorithmParameterException e) { -// throw new IOException(e); -// } - return null; + final BytesReference dataDecryptionKeyBytes = Streams.readFully(this.encryptionMetadataBlobContainer.readBlob(blobName)); + final SecretKey dataDecryptionKey; + try { + dataDecryptionKey = unwrapKey(BytesReference.toBytes(dataDecryptionKeyBytes), this.masterSecretKey); + } catch (NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException e) { + throw new IOException(e); + } + return GCMPacketsCipherInputStream.getDecryptor(this.delegatedBlobContainer.readBlob(blobName), dataDecryptionKey, NONCE, + BC_FIPS_PROVIDER); } @Override public void writeBlob(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) throws IOException { -// try { -// SecretKey dataEncryptionKey = generateRandomSecretKey(); -// byte[] wrappedDataEncryptionKey = wrapKey(dataEncryptionKey, this.masterSecretKey); -// try (InputStream stream = new ByteArrayInputStream(wrappedDataEncryptionKey)) { -// this.encryptionMetadataBlobContainer.writeBlob(blobName, stream, wrappedDataEncryptionKey.length, failIfAlreadyExists); -// } -// Cipher cipher = Cipher.getInstance(ENCRYPTION_MODE, BC_FIPS_PROV); -// cipher.init(Cipher.ENCRYPT_MODE, dataEncryptionKey, ivParameterSpec); -// cipher.update() -// this.delegatedBlobContainer.writeBlob(blobName, new CipherInputStream(inputStream, cipher), blobSize + GCM_TAG_BYTES_LENGTH, -// failIfAlreadyExists); -// } catch (NoSuchAlgorithmException | InvalidKeyException | NoSuchPaddingException | IllegalBlockSizeException -// | InvalidAlgorithmParameterException e) { -// throw new IOException(e); -// } + final byte[] wrappedDataEncryptionKey; + final SecretKey dataEncryptionKey; + try { + dataEncryptionKey = generateRandomSecretKey(); + wrappedDataEncryptionKey = wrapKey(dataEncryptionKey, this.masterSecretKey); + } catch (NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException | IllegalBlockSizeException e) { + throw new IOException(e); + } + try (InputStream stream = new ByteArrayInputStream(wrappedDataEncryptionKey)) { + this.encryptionMetadataBlobContainer.writeBlob(blobName, stream, wrappedDataEncryptionKey.length, failIfAlreadyExists); + } + final long encryptedSize = GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(blobSize); + try (InputStream encryptedInputStream = GCMPacketsCipherInputStream.getEncryptor(inputStream, dataEncryptionKey, NONCE, + BC_FIPS_PROVIDER)) { + this.delegatedBlobContainer.writeBlob(blobName, encryptedInputStream, encryptedSize, failIfAlreadyExists); + } } @Override @@ -263,7 +250,7 @@ public String name() { @Override public long length() { - return entry.getValue().length() - GCM_TAG_BYTES_LENGTH; + return GCMPacketsCipherInputStream.getDecryptionSizeFromCipherSize(entry.getValue().length()); } }); } @@ -274,7 +261,7 @@ public long length() { private static SecretKey generateSecretKeyFromPassword(char[] password) throws NoSuchAlgorithmException, InvalidKeySpecException { byte[] salt = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; // same salt for 1:1 password to key PBEKeySpec spec = new PBEKeySpec(password, salt, 65536, 256); - SecretKey tmp = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA256").generateSecret(spec); + SecretKey tmp = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA256", BC_FIPS_PROVIDER).generateSecret(spec); return new SecretKeySpec(tmp.getEncoded(), "AES"); } @@ -283,21 +270,21 @@ private static String keyId(SecretKey secretKey) { } private static SecretKey generateRandomSecretKey() throws NoSuchAlgorithmException { - KeyGenerator keyGen = KeyGenerator.getInstance("AES", BC_FIPS_PROV); - keyGen.init(256); + KeyGenerator keyGen = KeyGenerator.getInstance("AES", BC_FIPS_PROVIDER); + keyGen.init(256, SecureRandom.getInstance("DEFAULT", BC_FIPS_PROVIDER)); return keyGen.generateKey(); } private static byte[] wrapKey(SecretKey toWrap, SecretKey keyWrappingKey) throws NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException { - Cipher cipher = Cipher.getInstance("AESWrap"); + Cipher cipher = Cipher.getInstance("AESWrap", BC_FIPS_PROVIDER); cipher.init(Cipher.WRAP_MODE, keyWrappingKey); return cipher.wrap(toWrap); } private static SecretKey unwrapKey(byte[] toUnwrap, SecretKey keyEncryptionKey) throws NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException { - Cipher cipher = Cipher.getInstance("AESWrap"); + Cipher cipher = Cipher.getInstance("AESWrap", BC_FIPS_PROVIDER); cipher.init(Cipher.UNWRAP_MODE, keyEncryptionKey); return (SecretKey) cipher.unwrap(toUnwrap, "AES", Cipher.SECRET_KEY); } diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java index d3a57dff87421..1df6e928d9b60 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java @@ -16,6 +16,7 @@ import java.nio.ByteBuffer; import java.security.GeneralSecurityException; import java.security.Provider; +import java.util.Objects; import static javax.crypto.Cipher.DECRYPT_MODE; import static javax.crypto.Cipher.ENCRYPT_MODE; @@ -189,18 +190,18 @@ public static GCMPacketsCipherInputStream getDecryptor(InputStream in, SecretKey return getDecryptor(in, secretKey, nonce, new BouncyCastleFipsProvider()); } - public static int getEncryptionSizeFromPlainSize(int size) { + public static long getEncryptionSizeFromPlainSize(long size) { if (size < 0) { throw new IllegalArgumentException(); } return (size / PACKET_SIZE_IN_BYTES) * (ENCRYPTED_PACKET_SIZE_IN_BYTES) + (size % PACKET_SIZE_IN_BYTES) + GCM_TAG_SIZE_IN_BYTES; } - public static int getDecryptionSizeFromCipherSize(int size) { + public static long getDecryptionSizeFromCipherSize(long size) { if (size < GCM_TAG_SIZE_IN_BYTES) { throw new IllegalArgumentException(); } - int plainSize = (size / (ENCRYPTED_PACKET_SIZE_IN_BYTES)) * PACKET_SIZE_IN_BYTES; + long plainSize = (size / (ENCRYPTED_PACKET_SIZE_IN_BYTES)) * PACKET_SIZE_IN_BYTES; if (size % ENCRYPTED_PACKET_SIZE_IN_BYTES < GCM_TAG_SIZE_IN_BYTES) { throw new IllegalArgumentException(); } @@ -208,8 +209,8 @@ public static int getDecryptionSizeFromCipherSize(int size) { } private GCMPacketsCipherInputStream(InputStream in, SecretKey secretKey, int mode, long packetIndex, int nonce, Provider provider) { - super(in); - this.secretKey = secretKey; + super(Objects.requireNonNull(in)); + this.secretKey = Objects.requireNonNull(secretKey); this.mode = mode; this.packetIndex = packetIndex; this.provider = provider; @@ -233,7 +234,7 @@ private void reinitPacketCipher() throws GeneralSecurityException { cipher = Cipher.getInstance(GCM_ENCRYPTION_SCHEME); } // construct IV and increment packet index - packetIV.putLong(0, packetIndex++); + packetIV.putLong(0, Math.incrementExact(packetIndex)); GCMParameterSpec gcmParameterSpec = new GCMParameterSpec(GCM_TAG_SIZE_IN_BYTES * Byte.SIZE, packetIV.array()); cipher.init(mode, secretKey, gcmParameterSpec); packetCipher = cipher; diff --git a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java index bb8bf6ffd980d..4a829aa597b9b 100644 --- a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java +++ b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java @@ -80,8 +80,8 @@ public void testEncryptDecryptMultipleOfPacketSize() throws Exception { public void testMarkAndResetAtBeginningForEncryption() throws Exception { testMarkAndResetToSameOffsetForEncryption(0); - testMarkAndResetToSameOffsetForEncryption(GCMPacketsCipherInputStream. - getEncryptionSizeFromPlainSize(GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES)); + testMarkAndResetToSameOffsetForEncryption(Math.toIntExact(GCMPacketsCipherInputStream. + getEncryptionSizeFromPlainSize(GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES))); } public void testMarkAndResetFirstPacketForEncryption() throws Exception { @@ -111,7 +111,7 @@ public void testMarkAndResetCrawlForEncryption() throws Exception { secretKey, nonce, bcFipsProvider)) { cipherInputStream.mark(Integer.MAX_VALUE); for (int i = 0; i < GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length); i++) { - int skipSize = randomIntBetween(1, GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length) - i); + int skipSize = randomIntBetween(1, Math.toIntExact(GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length)) - i); // skip bytes cipherInputStream.readNBytes(skipSize); cipherInputStream.reset(); @@ -143,7 +143,7 @@ public void testMarkAndResetStepInRewindBuffer() throws Exception { int position4 = randomIntBetween(position3 + 1, 2 * ENCRYPTED_PACKET_SIZE_IN_BYTES - 2); int position5 = randomIntBetween(position4 + 1, 2 * ENCRYPTED_PACKET_SIZE_IN_BYTES - 1); int position6 = 2 * ENCRYPTED_PACKET_SIZE_IN_BYTES; - int position7 = GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length); + int position7 = Math.toIntExact(GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length)); // skip position1 bytes cipherInputStream.readNBytes(position1); // mark position1 @@ -292,7 +292,7 @@ secretKey, new Random().nextInt(), bcFipsProvider)) { cipherInputStream.reset(); // read/skip more than (encrypted) packet size, but less than the full stream skipSize = randomIntBetween(ENCRYPTED_PACKET_SIZE_IN_BYTES, - GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length) - 1 - offset); + Math.toIntExact(GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize((long)length)) - 1 - offset); byte[] secondPassEncryption = cipherInputStream.readNBytes(skipSize); assertTrue(Arrays.equals(firstPassEncryption, 0, firstPassEncryption.length, secondPassEncryption, 0, firstPassEncryption.length)); @@ -304,7 +304,7 @@ secretKey, new Random().nextInt(), bcFipsProvider)) { thirdPassEncryption = cipherInputStream.readAllBytes(); } else { thirdPassEncryption = - cipherInputStream.readNBytes(GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length) - offset); + cipherInputStream.readNBytes(Math.toIntExact(GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize((long)length)) - offset); } assertTrue(Arrays.equals(secondPassEncryption, 0, secondPassEncryption.length, thirdPassEncryption, 0, secondPassEncryption.length)); @@ -312,7 +312,7 @@ secretKey, new Random().nextInt(), bcFipsProvider)) { cipherInputStream.reset(); // read/skip more than (encrypted) packet size, but less than the full stream skipSize = randomIntBetween(ENCRYPTED_PACKET_SIZE_IN_BYTES, - GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length) - 1 - offset); + Math.toIntExact(GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize((long)length)) - 1 - offset); byte[] fourthPassEncryption = cipherInputStream.readNBytes(skipSize); assertTrue(Arrays.equals(fourthPassEncryption, 0, fourthPassEncryption.length, thirdPassEncryption, 0, fourthPassEncryption.length)); @@ -361,9 +361,9 @@ private SecretKey generateSecretKey() throws Exception { } // read "adversarily" in small random pieces - private ByteArrayOutputStream readAllInputStream(InputStream inputStream, int size) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(size); - byte[] temp = new byte[randomIntBetween(1, size != 0 ? size : 1)]; + private ByteArrayOutputStream readAllInputStream(InputStream inputStream, long size) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(Math.toIntExact(size)); + byte[] temp = new byte[randomIntBetween(1, size != 0 ? Math.toIntExact(size) : 1)]; do { int bytesRead = inputStream.read(temp, 0, randomIntBetween(1, temp.length)); if (bytesRead == -1) { @@ -378,7 +378,7 @@ private ByteArrayOutputStream readAllInputStream(InputStream inputStream, int si baos.write(singleByte); } } while (true); - assertThat(baos.size(), Matchers.is(size)); + assertThat(baos.size(), Matchers.is(Math.toIntExact(size))); return baos; } } From afafe762e5f7e90c0db14ec4852dab4ea457f829 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Fri, 18 Oct 2019 00:31:02 +0300 Subject: [PATCH 16/29] Check passes, no javadoc, no integ tests --- .../plugin/repository-encrypted/build.gradle | 15 +++++++++++ .../GCMPacketsCipherInputStream.java | 5 ++++ .../GCMPacketsCipherInputStreamTests.java | 25 +++++++++++-------- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/repository-encrypted/build.gradle b/x-pack/plugin/repository-encrypted/build.gradle index 7e0014987d934..3773140c02c64 100644 --- a/x-pack/plugin/repository-encrypted/build.gradle +++ b/x-pack/plugin/repository-encrypted/build.gradle @@ -8,6 +8,21 @@ esplugin { extendedPlugins = ['x-pack-core'] } +thirdPartyAudit { + ignoreViolations ( + 'org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider$CoreSecureRandom', + 'org.bouncycastle.jcajce.provider.ProvSunTLSKDF', + 'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$BaseTLSKeyGeneratorSpi', + 'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSKeyMaterialGenerator', + 'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSKeyMaterialGenerator$2', + 'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSMasterSecretGenerator', + 'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSMasterSecretGenerator$2', + 'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSPRFKeyGenerator', + 'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSRsaPreMasterSecretGenerator', + 'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSRsaPreMasterSecretGenerator$2', + ) +} + dependencies { compile "org.bouncycastle:bc-fips:1.0.1" compile "org.bouncycastle:bcpkix-fips:1.0.3" diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java index 1df6e928d9b60..a5f4167c0e787 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java @@ -1,3 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ package org.elasticsearch.repositories.encrypted; import org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider; diff --git a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java index 4a829aa597b9b..c90f5f3d821ce 100644 --- a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java +++ b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java @@ -1,6 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ package org.elasticsearch.repositories.encrypted; import org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider; +import org.elasticsearch.common.Randomness; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; import org.junit.Before; @@ -36,7 +42,7 @@ static void setupProvider() { return null; }); testPlaintextArray = new byte[TEST_ARRAY_SIZE]; - new Random().nextBytes(testPlaintextArray); + Randomness.get().nextBytes(testPlaintextArray); } @Before @@ -99,7 +105,7 @@ public void testMarkAndResetRandomSecondPacketForEncryption() throws Exception { public void testMarkAndResetCrawlForEncryption() throws Exception { int length = GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES; int startIndex = randomIntBetween(0, testPlaintextArray.length - length); - int nonce = new Random().nextInt(); + int nonce = Randomness.get().nextInt(); byte[] ciphertextBytes; try (InputStream cipherInputStream = GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), @@ -127,7 +133,7 @@ public void testMarkAndResetCrawlForEncryption() throws Exception { public void testMarkAndResetStepInRewindBuffer() throws Exception { int length = 2 * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES; int startIndex = randomIntBetween(0, testPlaintextArray.length - length); - int nonce = new Random().nextInt(); + int nonce = Randomness.get().nextInt(); byte[] ciphertextBytes; try (InputStream cipherInputStream = GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), @@ -214,10 +220,10 @@ public void testMarkAndResetStepInRewindBuffer() throws Exception { } public void testDecryptionFails() throws Exception { - Random random = new Random(); + Random random = Randomness.get(); int length = randomIntBetween(0, READ_BUFFER_SIZE_IN_BYTES); int startIndex = randomIntBetween(0, testPlaintextArray.length - length); - int nonce = new Random().nextInt(); + int nonce = random.nextInt(); byte[] ciphertextBytes; try (InputStream cipherInputStream = GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), @@ -280,7 +286,7 @@ private void testMarkAndResetToSameOffsetForEncryption(int offset) throws Except int startIndex = randomIntBetween(0, testPlaintextArray.length - length); try (InputStream cipherInputStream = GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), - secretKey, new Random().nextInt(), bcFipsProvider)) { + secretKey, Randomness.get().nextInt(), bcFipsProvider)) { // skip offset bytes cipherInputStream.readNBytes(offset); // mark after offset @@ -303,8 +309,8 @@ secretKey, new Random().nextInt(), bcFipsProvider)) { if (randomBoolean()) { thirdPassEncryption = cipherInputStream.readAllBytes(); } else { - thirdPassEncryption = - cipherInputStream.readNBytes(Math.toIntExact(GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize((long)length)) - offset); + thirdPassEncryption = cipherInputStream.readNBytes( + Math.toIntExact(GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize((long) length)) - offset); } assertTrue(Arrays.equals(secondPassEncryption, 0, secondPassEncryption.length, thirdPassEncryption, 0, secondPassEncryption.length)); @@ -327,8 +333,7 @@ secretKey, new Random().nextInt(), bcFipsProvider)) { } private void testEncryptDecryptRandomOfLength(int length, SecretKey secretKey) throws Exception { - Random random = new Random(); - int nonce = random.nextInt(); + int nonce = Randomness.get().nextInt(); ByteArrayOutputStream cipherTextOutput; ByteArrayOutputStream plainTextOutput; int startIndex = randomIntBetween(0, testPlaintextArray.length - length); From 780f8a1d501dd343717e837e79226ecbbcde6071 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Fri, 18 Oct 2019 01:39:51 +0300 Subject: [PATCH 17/29] Works! --- .../encrypted/EncryptedRepository.java | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java index 6b561215eec85..78e4987d921db 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java @@ -6,6 +6,7 @@ package org.elasticsearch.repositories.encrypted; +import org.bouncycastle.crypto.fips.FipsUnapprovedOperationError; import org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider; import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.Strings; @@ -35,8 +36,10 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.security.AccessController; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; +import java.security.PrivilegedAction; import java.security.SecureRandom; import java.security.spec.InvalidKeySpecException; import java.util.HashMap; @@ -45,7 +48,13 @@ public class EncryptedRepository extends BlobStoreRepository { - private static final BouncyCastleFipsProvider BC_FIPS_PROVIDER = new BouncyCastleFipsProvider(); + private static BouncyCastleFipsProvider BC_FIPS_PROVIDER; + static { + AccessController.doPrivileged((PrivilegedAction) () -> { + EncryptedRepository.BC_FIPS_PROVIDER = new BouncyCastleFipsProvider(); + return null; + }); + } // TODO change this and make it (secure) randomly generated for each blob private static final int NONCE = 5318008; @@ -261,8 +270,15 @@ public long length() { private static SecretKey generateSecretKeyFromPassword(char[] password) throws NoSuchAlgorithmException, InvalidKeySpecException { byte[] salt = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; // same salt for 1:1 password to key PBEKeySpec spec = new PBEKeySpec(password, salt, 65536, 256); - SecretKey tmp = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA256", BC_FIPS_PROVIDER).generateSecret(spec); - return new SecretKeySpec(tmp.getEncoded(), "AES"); + try { + SecretKey tmp = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA256", BC_FIPS_PROVIDER).generateSecret(spec); + return new SecretKeySpec(tmp.getEncoded(), "AES"); + } catch (FipsUnapprovedOperationError e) { + // password must be at least 112 bits + // wrap the error into an exception because the exception is better handled up the stack + // TODO don't do this wrapping + throw new RuntimeException(e); + } } private static String keyId(SecretKey secretKey) { From 51273d57367d8e3c3e4244df137d48beb0914802 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Fri, 18 Oct 2019 02:26:20 +0300 Subject: [PATCH 18/29] GCMPacketsInputStream javadoc --- .../GCMPacketsCipherInputStream.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java index a5f4167c0e787..0969c903c6f30 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java @@ -27,6 +27,32 @@ import static javax.crypto.Cipher.ENCRYPT_MODE; /** + * Given an {@code InputStream} and a {@code SecretKey}, creates a wrapper {@code InputStream} that encrypts/decrypts the content of the + * original stream. The original stream must not be otherwise used; it will be closed when the encryptor/decryptor stream is closed. + * + * The method of encryption is AES/GCM/NOPadding. The GCM mode is a form of authenticated encryption, meaning it offers authenticity in + * addition to the expected confidentiality. In other words, during decryption it verifies that the ciphertext being decrypted has not + * been tampered with. + * + * During encryption the source input stream is processed piece-wise and a packet (consisting of multiple pieces) of at most {@code + * GCMPacketsCipherInputStream#PACKET_SIZE_IN_BYTES} bytes size is encrypted and authenticated independently of the other packets. All + * packets in the same stream are encrypted with the same secret key, but a different IV, monotonically increasing with the packet index. + * Consequently, each packet has its own authentication tag appended, even the empty packet (all packets are the same size, but the last + * one can be of any size). The resulting ciphertext has a larger size than the source plaintext. + * + * Decryption also validates the authentication tag. It is important that the {@code Cipher} used during decryption, which is returned by + * the {@code Provider} parameter, NOT internally cache pieces of ciphertext, without releasing the decrypted plaintext, until it + * validates the associated authentication tag. Failure to comply to this requirement, will choke (throw {@code IllegalStateException}) the + * decryption stream, because the ciphertext is processed piece wise, the complete packet is not available fully at one moment. + * + * The resulting decrypted stream will return possibly un-authenticated content, but it is guaranteed that an {@code IOException} is + * thrown, at the latest when the stream has been exhausted ({@code InputStream#read} return {@code -1}) if the ciphertext has been altered. + * + * Both the encrypting and decrypting streams support {@code InputStream#mark} and {@code InputStream#reset}. Because GCM processing cannot + * be reset to a previous state, it only "goes forward", a mark call during the processing of a packet will buffer the processed bytes + * until the next packet boundary. A reset call will pick up any buffered data from a partially processed packet, and re-encrypt the + * following packets (the following packets of a mark call are not buffered, they are re-encrypted). + * * This is obviously NOT thread-safe. */ public class GCMPacketsCipherInputStream extends FilterInputStream { From 4bc131563606faa3dafa50d06139c1c2d77f7ed9 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 31 Oct 2019 19:49:14 +0200 Subject: [PATCH 19/29] Rough EncryptedBlobMetadata --- .../encrypted/EncryptedBlobMetadata.java | 181 ++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedBlobMetadata.java diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedBlobMetadata.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedBlobMetadata.java new file mode 100644 index 0000000000000..669df5a181599 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedBlobMetadata.java @@ -0,0 +1,181 @@ +package org.elasticsearch.repositories.encrypted; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + +public class EncryptedBlobMetadata implements ToXContentObject { + + private static ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("encrypted_blob_metadata", false, + args -> { + @SuppressWarnings("unchecked") Integer version = (Integer) args[0]; + @SuppressWarnings("unchecked") String wrappedDataEncryptionKey = (String) args[1]; + @SuppressWarnings("unchecked") Integer maximumPacketSizeInBytes = (Integer) args[2]; + @SuppressWarnings("unchecked") Integer authenticationTagSizeInBytes = (Integer) args[3]; + @SuppressWarnings("unchecked") List packetsInfo = (List) args[4]; + return new EncryptedBlobMetadata(version, Base64.getDecoder().decode(wrappedDataEncryptionKey), + maximumPacketSizeInBytes, authenticationTagSizeInBytes, packetsInfo); + } + ); + + static { + PARSER.declareInt(constructorArg(), Fields.VERSION); + PARSER.declareString(constructorArg(), Fields.WRAPPED_DEK); + PARSER.declareInt(constructorArg(), Fields.MAXIMUM_PACKET_SIZE); + PARSER.declareInt(constructorArg(), Fields.AUTH_TAG_SIZE); + PARSER.declareObjectArray(constructorArg(), (p, c) -> PacketInfo.fromXContent(p), Fields.PACKETS_INFO); + } + + public static EncryptedBlobMetadata fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + private final int version; + private final byte[] wrappedDataEncryptionKey; + private final int maximumPacketSizeInBytes; + private final int authenticationTagSizeInBytes; + private List packetsInfo; + + public EncryptedBlobMetadata(int version, byte[] wrappedDataEncryptionKey, int maximumPacketSizeInBytes, + int authenticationTagSizeInBytes, List packetsInfo) { + this.version = version; + this.wrappedDataEncryptionKey = wrappedDataEncryptionKey; + this.maximumPacketSizeInBytes = maximumPacketSizeInBytes; + this.authenticationTagSizeInBytes = authenticationTagSizeInBytes; + this.packetsInfo = Collections.unmodifiableList(packetsInfo); + } + + public byte[] getWrappedDataEncryptionKey() { + return wrappedDataEncryptionKey; + } + + public int getVersion() { + return version; + } + + public int getMaximumPacketSizeInBytes() { + return maximumPacketSizeInBytes; + } + + public List getPacketsInfo() { + return packetsInfo; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Fields.VERSION.getPreferredName(), version); + builder.field(Fields.WRAPPED_DEK.getPreferredName(), Base64.getEncoder().encodeToString(wrappedDataEncryptionKey)); + builder.field(Fields.MAXIMUM_PACKET_SIZE.getPreferredName(), maximumPacketSizeInBytes); + builder.field(Fields.AUTH_TAG_SIZE.getPreferredName(), authenticationTagSizeInBytes); + builder.startArray(Fields.PACKETS_INFO.getPreferredName()); + for (PacketInfo packetInfo : packetsInfo) { + packetInfo.toXContent(builder, ToXContent.EMPTY_PARAMS); + } + builder.endArray(); + builder.endObject(); + return null; + } + + private interface Fields { + ParseField VERSION = new ParseField("version"); + ParseField WRAPPED_DEK = new ParseField("wrappedDataEncryptionKey"); + ParseField MAXIMUM_PACKET_SIZE = new ParseField("maximumPacketSizeInBytes"); + ParseField AUTH_TAG_SIZE = new ParseField("authenticationTagSizeInBytes"); + ParseField PACKETS_INFO = new ParseField("packetsInfo"); + } + + static class PacketInfo implements ToXContentObject { + + private static ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("packet_info", false, + args -> { + @SuppressWarnings("unchecked") String iv = (String) args[0]; + @SuppressWarnings("unchecked") String authenticationTag = (String) args[1]; + @SuppressWarnings("unchecked") Integer sizeInBytes = (Integer) args[2]; + return new PacketInfo(Base64.getDecoder().decode(iv), Base64.getDecoder().decode(authenticationTag), + sizeInBytes); + } + ); + + static { + PARSER.declareString(constructorArg(), Fields.IV); + PARSER.declareString(constructorArg(), Fields.AUTHENTICATION_TAG); + PARSER.declareInt(constructorArg(), Fields.SIZE_IN_BYTES); + } + + static PacketInfo fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + private final byte[] iv; + private final byte[] authenticationTag; + private final int sizeInBytes; + + PacketInfo(byte[] iv, byte[] authenticationTag, int sizeInBytes) { + this.iv = iv; + this.authenticationTag = authenticationTag; + this.sizeInBytes = sizeInBytes; + } + + byte[] getIv() { + return iv; + } + + byte[] getAuthenticationTag() { + return authenticationTag; + } + + int getSizeInBytes() { + return sizeInBytes; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PacketInfo that = (PacketInfo) o; + return sizeInBytes == that.sizeInBytes && + Arrays.equals(iv, that.iv) && + Arrays.equals(authenticationTag, that.authenticationTag); + } + + @Override + public int hashCode() { + int result = Objects.hash(sizeInBytes); + result = 31 * result + Arrays.hashCode(iv); + result = 31 * result + Arrays.hashCode(authenticationTag); + return result; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Fields.IV.getPreferredName(), Base64.getEncoder().encodeToString(iv)); + builder.field(Fields.AUTHENTICATION_TAG.getPreferredName(), Base64.getEncoder().encodeToString(authenticationTag)); + builder.field(Fields.SIZE_IN_BYTES.getPreferredName(), sizeInBytes); + builder.endObject(); + return null; + } + + private interface Fields { + ParseField IV = new ParseField("iv"); + ParseField AUTHENTICATION_TAG = new ParseField("authenticationTag"); + ParseField SIZE_IN_BYTES = new ParseField("sizeInBytes"); + } + } + +} From 6b8a6f369c59651143d131b35338cbf3c293b195 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Sun, 17 Nov 2019 13:22:19 +0200 Subject: [PATCH 20/29] GCMPacketsEncryptor close/mark/reset TODO --- .../GCMPacketsEncryptorInputStream.java | 233 ++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java new file mode 100644 index 0000000000000..15b6947cbca5c --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java @@ -0,0 +1,233 @@ +package org.elasticsearch.repositories.encrypted; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; +import javax.crypto.ShortBufferException; +import javax.crypto.spec.GCMParameterSpec; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class GCMPacketsEncryptorInputStream extends FilterInputStream { + + private static final int GCM_TAG_SIZE_IN_BYTES = 16; + private static final int GCM_IV_SIZE_IN_BYTES = 12; + private static final String GCM_ENCRYPTION_SCHEME = "AES/GCM/NoPadding"; + private static final int AES_BLOCK_SIZE_IN_BYTES = 128; + + private final Logger logger = LogManager.getLogger(getClass()); + private final int maxPacketSizeInBytes; + private final byte[] packetTrailByteBuffer; + private final SecretKey secretKey; + private final IvRandomGenerator ivGenerator; + private final List packetInfoList; + + private int bytesRemainingInPacket; + private byte[] packetIv; + private Cipher packetCipher; + private boolean closed; + + protected GCMPacketsEncryptorInputStream(InputStream in, SecretKey secretKey, int maxPacketSizeInBytes) throws IOException { + super(in); + this.maxPacketSizeInBytes = maxPacketSizeInBytes; + this.packetTrailByteBuffer = new byte[AES_BLOCK_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES]; + this.secretKey = secretKey; + this.ivGenerator = new IvRandomGenerator(); + this.packetInfoList = new ArrayList<>(); + this.bytesRemainingInPacket = maxPacketSizeInBytes; + this.packetIv = ivGenerator.newUniqueIv(); + this.packetCipher = getPacketEncryptionCipher(secretKey, packetIv); + this.closed = false; + } + + @Override + public int read() throws IOException { + byte[] b = new byte[1]; + int bytesRead = read(b, 0, 1); + if (bytesRead == -1) { + return -1; + } + if (bytesRead != 1) { + throw new IllegalStateException(); + } + return (int) b[0]; + } + + @Override + public int read(byte b[], int off, int len) throws IOException { + int maxReadSize = getReadSize(len); + int readSize = in.readNBytes(b, off, maxReadSize); + assert readSize >= 0 : "readNBytes does not return -1 on end-of-stream"; + if (readSize == 0) { + if (maxReadSize == 0) { + // 0 bytes were requested + return 0; + } + // end of filtered input stream + assert maxReadSize > 0; + assert in.read() == -1 : "readNBytes returned no bytes but it's not the end-of-stream"; + return -1; + } + bytesRemainingInPacket -= readSize; + final int encryptedSize; + try { + // in-place encryption + encryptedSize = packetCipher.update(b, off, readSize, b, off); + } catch (ShortBufferException e) { + throw new IllegalStateException(e); + } + if (bytesRemainingInPacket == 0 || readSize % AES_BLOCK_SIZE_IN_BYTES != 0) { + final byte[] authenticationTag; + if (encryptedSize == readSize) { + try { + authenticationTag = packetCipher.doFinal(); + } catch (IllegalBlockSizeException | BadPaddingException e) { + throw new IOException(e); + } + } else { + if (readSize - encryptedSize >= AES_BLOCK_SIZE_IN_BYTES) { + throw new IllegalStateException(); + } + int trailAndTagSize = 0; + try { + trailAndTagSize = packetCipher.doFinal(packetTrailByteBuffer, 0); + } catch (IllegalBlockSizeException | ShortBufferException | BadPaddingException e) { + throw new IOException(e); + } + if (encryptedSize + trailAndTagSize != readSize + GCM_TAG_SIZE_IN_BYTES) { + throw new IllegalStateException(); + } + // copy the remaining packet trail bytes + System.arraycopy(packetTrailByteBuffer, 0, b, off + encryptedSize, trailAndTagSize - GCM_TAG_SIZE_IN_BYTES); + authenticationTag = Arrays.copyOfRange(packetTrailByteBuffer, trailAndTagSize - GCM_TAG_SIZE_IN_BYTES, trailAndTagSize); + } + if (authenticationTag.length != GCM_TAG_SIZE_IN_BYTES) { + throw new IllegalStateException(); + } + packetInfoList.add(new EncryptedBlobMetadata.PacketInfo(packetIv, authenticationTag, maxPacketSizeInBytes - bytesRemainingInPacket)); + bytesRemainingInPacket = maxPacketSizeInBytes; + packetIv = ivGenerator.newUniqueIv(); + packetCipher = getPacketEncryptionCipher(secretKey, packetIv); + return readSize; + } else { + if (encryptedSize != readSize) { + throw new IllegalStateException(); + } + return readSize; + } + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + closed = true; + in.close(); + // Throw away the unprocessed data and throw no crypto exceptions. + // Normally the GCM cipher is fully readed before closing, so any authentication + // exceptions would occur while reading. + if (false == done) { + done = true; + try { + packetCipher.doFinal(); + } catch (BadPaddingException | IllegalBlockSizeException ex) { + // Catch exceptions as the rest of the stream is unused. + } + } + } + + @Override + public void mark(int readLimit) { + } + + @Override + public void reset() throws IOException { + } + + private static Cipher getPacketEncryptionCipher(SecretKey secretKey, byte[] packetIv) throws IOException { + GCMParameterSpec gcmParameterSpec = new GCMParameterSpec(GCM_TAG_SIZE_IN_BYTES * Byte.SIZE, packetIv); + try { + Cipher packetCipher = Cipher.getInstance(GCM_ENCRYPTION_SCHEME); + packetCipher.init(Cipher.ENCRYPT_MODE, secretKey, gcmParameterSpec); + return packetCipher; + } catch (NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException | InvalidAlgorithmParameterException e) { + throw new IOException(e); + } + } + + private int getReadSize(int len) { + if (bytesRemainingInPacket <= 0) { + throw new IllegalStateException(); + } + if (len < 0) { + throw new IndexOutOfBoundsException(); + } + if (len == 0) { + return 0; + } + int maxReadSize = Math.min(len, bytesRemainingInPacket); + int readSize = (maxReadSize / AES_BLOCK_SIZE_IN_BYTES) * AES_BLOCK_SIZE_IN_BYTES; + if (readSize != 0) { + return readSize; + } + assert maxReadSize < AES_BLOCK_SIZE_IN_BYTES; + if (maxReadSize == len) { + logger.warn("Reading [" + len + "] bytes, which is less than [" + AES_BLOCK_SIZE_IN_BYTES + "], is terribly inefficient."); + } + return maxReadSize; + } + + static class IvRandomGenerator { + + private final Map> generatedIvs; + private final SecureRandom secureRandom; + + IvRandomGenerator() { + generatedIvs = new HashMap<>(); + secureRandom = new SecureRandom(); + } + + byte[] newUniqueIv() { + byte[] uniqueIv = newUniqueIv(3); + if (uniqueIv == null) { + throw new IllegalStateException("Secure random returns many similar values"); + } + return uniqueIv; + } + + private byte[] newUniqueIv(int retryCount) { + if (retryCount <= 0) { + return null; + } + long part1 = secureRandom.nextLong(); + Set part2Set = generatedIvs.computeIfAbsent(part1, k -> new HashSet<>()); + int part2 = secureRandom.nextInt(); + if (false == part2Set.add(part2)) { + return newUniqueIv(retryCount - 1); + } + ByteBuffer uniqueIv = ByteBuffer.allocate(GCM_IV_SIZE_IN_BYTES); + uniqueIv.putLong(part1); + uniqueIv.putInt(part2); + return uniqueIv.array(); + } + } +} From 3c97079845496ea9ceb968428f56e2a06ccea1f0 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Sun, 17 Nov 2019 14:56:36 +0200 Subject: [PATCH 21/29] Close/mark/reset done! --- .../GCMPacketsEncryptorInputStream.java | 52 ++++++++++++++----- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java index 15b6947cbca5c..67b8b83e24cb1 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java @@ -13,6 +13,7 @@ import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.security.InvalidAlgorithmParameterException; import java.security.InvalidKeyException; @@ -26,6 +27,8 @@ import java.util.Map; import java.util.Set; + +// not thread-safe public class GCMPacketsEncryptorInputStream extends FilterInputStream { private static final int GCM_TAG_SIZE_IN_BYTES = 16; @@ -44,6 +47,7 @@ public class GCMPacketsEncryptorInputStream extends FilterInputStream { private byte[] packetIv; private Cipher packetCipher; private boolean closed; + private int markPacketIndex; protected GCMPacketsEncryptorInputStream(InputStream in, SecretKey secretKey, int maxPacketSizeInBytes) throws IOException { super(in); @@ -56,6 +60,7 @@ protected GCMPacketsEncryptorInputStream(InputStream in, SecretKey secretKey, in this.packetIv = ivGenerator.newUniqueIv(); this.packetCipher = getPacketEncryptionCipher(secretKey, packetIv); this.closed = false; + this.markPacketIndex = -1; } @Override @@ -122,10 +127,7 @@ public int read(byte b[], int off, int len) throws IOException { if (authenticationTag.length != GCM_TAG_SIZE_IN_BYTES) { throw new IllegalStateException(); } - packetInfoList.add(new EncryptedBlobMetadata.PacketInfo(packetIv, authenticationTag, maxPacketSizeInBytes - bytesRemainingInPacket)); - bytesRemainingInPacket = maxPacketSizeInBytes; - packetIv = ivGenerator.newUniqueIv(); - packetCipher = getPacketEncryptionCipher(secretKey, packetIv); + finishPacket(authenticationTag); return readSize; } else { if (encryptedSize != readSize) { @@ -135,6 +137,14 @@ public int read(byte b[], int off, int len) throws IOException { } } + private void finishPacket(byte[] authenticationTag) throws IOException { + packetInfoList.add(new EncryptedBlobMetadata.PacketInfo(packetIv, authenticationTag, + maxPacketSizeInBytes - bytesRemainingInPacket)); + bytesRemainingInPacket = maxPacketSizeInBytes; + packetIv = ivGenerator.newUniqueIv(); + packetCipher = getPacketEncryptionCipher(secretKey, packetIv); + } + @Override public void close() throws IOException { if (closed) { @@ -142,25 +152,43 @@ public void close() throws IOException { } closed = true; in.close(); - // Throw away the unprocessed data and throw no crypto exceptions. - // Normally the GCM cipher is fully readed before closing, so any authentication - // exceptions would occur while reading. - if (false == done) { - done = true; + if (bytesRemainingInPacket < maxPacketSizeInBytes) { + final byte[] authenticationTag; try { - packetCipher.doFinal(); - } catch (BadPaddingException | IllegalBlockSizeException ex) { - // Catch exceptions as the rest of the stream is unused. + authenticationTag = packetCipher.doFinal(); + } catch (IllegalBlockSizeException | BadPaddingException e) { + throw new IOException(e); } + finishPacket(authenticationTag); } } @Override public void mark(int readLimit) { + in.mark(readLimit); + // finish in-progress packet + if (bytesRemainingInPacket < maxPacketSizeInBytes) { + try { + byte[] authenticationTag = packetCipher.doFinal(); + finishPacket(authenticationTag); + } catch (IllegalBlockSizeException | BadPaddingException e) { + throw new UncheckedIOException(new IOException(e)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + markPacketIndex = packetInfoList.size(); } @Override public void reset() throws IOException { + in.reset(); + // discard packets after mark point + packetInfoList.subList(markPacketIndex, packetInfoList.size()).clear(); + // reinstantiate packetCipher + bytesRemainingInPacket = maxPacketSizeInBytes; + packetIv = ivGenerator.newUniqueIv(); + packetCipher = getPacketEncryptionCipher(secretKey, packetIv); } private static Cipher getPacketEncryptionCipher(SecretKey secretKey, byte[] packetIv) throws IOException { From 5dc188e56465d15f70ccc5273ff18d34528b46c9 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Sun, 17 Nov 2019 15:34:46 +0200 Subject: [PATCH 22/29] GCMPacketsEncryptorInputStream done! --- .../encrypted/EncryptedRepository.java | 1 + .../GCMPacketsEncryptorInputStream.java | 69 ++++++++++++------- 2 files changed, 44 insertions(+), 26 deletions(-) diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java index 78e4987d921db..88c69059c6494 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java @@ -57,6 +57,7 @@ public class EncryptedRepository extends BlobStoreRepository { } // TODO change this and make it (secure) randomly generated for each blob private static final int NONCE = 5318008; + private static final int ENCRYPTION_PROTOCOL_VERSION_NUMBER = 1; static final Setting.AffixSetting ENCRYPTION_PASSWORD_SETTING = Setting.affixKeySetting("repository.encrypted.", "password", key -> SecureSetting.secureString(key, null)); diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java index 67b8b83e24cb1..18aaa5eb7518b 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java @@ -27,7 +27,6 @@ import java.util.Map; import java.util.Set; - // not thread-safe public class GCMPacketsEncryptorInputStream extends FilterInputStream { @@ -63,21 +62,9 @@ protected GCMPacketsEncryptorInputStream(InputStream in, SecretKey secretKey, in this.markPacketIndex = -1; } - @Override - public int read() throws IOException { - byte[] b = new byte[1]; - int bytesRead = read(b, 0, 1); - if (bytesRead == -1) { - return -1; - } - if (bytesRead != 1) { - throw new IllegalStateException(); - } - return (int) b[0]; - } - @Override public int read(byte b[], int off, int len) throws IOException { + ensureOpen(); int maxReadSize = getReadSize(len); int readSize = in.readNBytes(b, off, maxReadSize); assert readSize >= 0 : "readNBytes does not return -1 on end-of-stream"; @@ -137,12 +124,17 @@ public int read(byte b[], int off, int len) throws IOException { } } - private void finishPacket(byte[] authenticationTag) throws IOException { - packetInfoList.add(new EncryptedBlobMetadata.PacketInfo(packetIv, authenticationTag, - maxPacketSizeInBytes - bytesRemainingInPacket)); - bytesRemainingInPacket = maxPacketSizeInBytes; - packetIv = ivGenerator.newUniqueIv(); - packetCipher = getPacketEncryptionCipher(secretKey, packetIv); + @Override + public int read() throws IOException { + byte[] b = new byte[1]; + int bytesRead = read(b, 0, 1); + if (bytesRead == -1) { + return -1; + } + if (bytesRead != 1) { + throw new IllegalStateException(); + } + return (int) b[0]; } @Override @@ -182,6 +174,7 @@ public void mark(int readLimit) { @Override public void reset() throws IOException { + ensureOpen(); in.reset(); // discard packets after mark point packetInfoList.subList(markPacketIndex, packetInfoList.size()).clear(); @@ -191,6 +184,27 @@ public void reset() throws IOException { packetCipher = getPacketEncryptionCipher(secretKey, packetIv); } + public List getEncryptionPacketMetadata() { + if (false == closed) { + throw new IllegalStateException(); + } + return List.copyOf(packetInfoList); + } + + void ensureOpen() throws IOException { + if (closed) { + throw new IOException("Stream closed"); + } + } + + private void finishPacket(byte[] authenticationTag) throws IOException { + packetInfoList.add(new EncryptedBlobMetadata.PacketInfo(packetIv, authenticationTag, + maxPacketSizeInBytes - bytesRemainingInPacket)); + bytesRemainingInPacket = maxPacketSizeInBytes; + packetIv = ivGenerator.newUniqueIv(); + packetCipher = getPacketEncryptionCipher(secretKey, packetIv); + } + private static Cipher getPacketEncryptionCipher(SecretKey secretKey, byte[] packetIv) throws IOException { GCMParameterSpec gcmParameterSpec = new GCMParameterSpec(GCM_TAG_SIZE_IN_BYTES * Byte.SIZE, packetIv); try { @@ -202,6 +216,13 @@ private static Cipher getPacketEncryptionCipher(SecretKey secretKey, byte[] pack } } + /** + * Tries to return a read size value such that it is smaller or equal to the requested {@code len}, does not exceed the remaining + * space in the current packet and, very important, is a multiple of {@link #AES_BLOCK_SIZE_IN_BYTES}. If the requested {@code len} + * or the remaining space in the current packet are smaller than {@link #AES_BLOCK_SIZE_IN_BYTES}, then their minimum is returned. + * + * @param len the requested read size + */ private int getReadSize(int len) { if (bytesRemainingInPacket <= 0) { throw new IllegalStateException(); @@ -235,16 +256,12 @@ static class IvRandomGenerator { } byte[] newUniqueIv() { - byte[] uniqueIv = newUniqueIv(3); - if (uniqueIv == null) { - throw new IllegalStateException("Secure random returns many similar values"); - } - return uniqueIv; + return newUniqueIv(5); } private byte[] newUniqueIv(int retryCount) { if (retryCount <= 0) { - return null; + throw new IllegalStateException("Secure random returns many similar values"); } long part1 = secureRandom.nextLong(); Set part2Set = generatedIvs.computeIfAbsent(part1, k -> new HashSet<>()); From cc9cbc4ffc15eab953906a0ae45764aa05eaf4e4 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Sun, 17 Nov 2019 23:07:54 +0200 Subject: [PATCH 23/29] GCMPacketsDecryptorInputStream Done! --- .../encrypted/EncryptedRepository.java | 5 + .../GCMPacketsDecryptorInputStream.java | 172 ++++++++++++++++++ .../GCMPacketsEncryptorInputStream.java | 56 +++--- 3 files changed, 204 insertions(+), 29 deletions(-) create mode 100644 x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsDecryptorInputStream.java diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java index 88c69059c6494..ab171e234e77b 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java @@ -48,6 +48,11 @@ public class EncryptedRepository extends BlobStoreRepository { + static final int GCM_TAG_SIZE_IN_BYTES = 16; + static final int GCM_IV_SIZE_IN_BYTES = 12; + static final String GCM_ENCRYPTION_SCHEME = "AES/GCM/NoPadding"; + static final int AES_BLOCK_SIZE_IN_BYTES = 128; + private static BouncyCastleFipsProvider BC_FIPS_PROVIDER; static { AccessController.doPrivileged((PrivilegedAction) () -> { diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsDecryptorInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsDecryptorInputStream.java new file mode 100644 index 0000000000000..f9d513ddb5f49 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsDecryptorInputStream.java @@ -0,0 +1,172 @@ +package org.elasticsearch.repositories.encrypted; + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; +import javax.crypto.ShortBufferException; +import javax.crypto.spec.GCMParameterSpec; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.util.List; + +public class GCMPacketsDecryptorInputStream extends FilterInputStream { + + private final int maxPacketSizeInBytes; + private final int authenticationTagSizeInBytes; + private final SecretKey secretKey; + private final List packetInfoList; + private final byte[] packetBuffer; + + private int packetIndex; + private int bufferStartOffset; + private int bufferEndOffset; + private boolean closed; + + protected GCMPacketsDecryptorInputStream(InputStream in, SecretKey secretKey, int maxPacketSizeInBytes, + int authenticationTagSizeInBytes, List packetInfoList) { + super(in); + this.secretKey = secretKey; + this.maxPacketSizeInBytes = maxPacketSizeInBytes; + this.authenticationTagSizeInBytes = authenticationTagSizeInBytes; + this.packetInfoList = packetInfoList; + this.packetBuffer = new byte[maxPacketSizeInBytes + authenticationTagSizeInBytes]; + this.packetIndex = 0; + this.bufferStartOffset = 0; + this.bufferEndOffset = 0; + this.closed = false; + } + + @Override + public int read() throws IOException { + if (bufferStartOffset >= bufferEndOffset) { + bufferEndOffset = readAndDecryptNextPacket(); + if (bufferEndOffset == -1) { + return -1; + } + bufferStartOffset = 0; + } + return packetBuffer[bufferStartOffset++]; + } + + @Override + public int read(byte b[], int off, int len) throws IOException { + if (off < 0 || len < 0 || len > b.length - off) { + throw new IndexOutOfBoundsException(); + } + if (len == 0) { + return 0; + } + if (bufferStartOffset >= bufferEndOffset) { + bufferEndOffset = readAndDecryptNextPacket(); + if (bufferEndOffset == -1) { + return -1; + } + bufferStartOffset = 0; + } + int readSize = Math.min(len, bufferEndOffset - bufferStartOffset); + System.arraycopy(packetBuffer, bufferStartOffset, b, off, readSize); + bufferStartOffset += readSize; + return readSize; + } + + @Override + public long skip(long n) throws IOException { + if (n == 0L) { + return 0L; + } + if (bufferStartOffset >= bufferEndOffset) { + bufferEndOffset = readAndDecryptNextPacket(); + if (bufferEndOffset == -1) { + return 0; + } + bufferStartOffset = 0; + } + int skipSize = Math.toIntExact(Math.min(n, bufferEndOffset - bufferStartOffset)); + bufferStartOffset += skipSize; + return skipSize; + } + + @Override + public int available() throws IOException { + return Math.max(0, bufferEndOffset - bufferStartOffset); + } + + @Override + public boolean markSupported() { + return false; + } + + @Override + public void mark(int readLimit) { + } + + @Override + public void reset() throws IOException { + throw new IOException("mark/reset not supported"); + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + closed = true; + in.close(); + } + + void ensureOpen() throws IOException { + if (closed) { + throw new IOException("Stream closed"); + } + } + + private int readAndDecryptNextPacket() throws IOException { + if (packetIndex == packetInfoList.size()) { + return -1; + } + EncryptedBlobMetadata.PacketInfo currentPacketInfo = packetInfoList.get(packetIndex++); + int packetSize = currentPacketInfo.getSizeInBytes(); + if (packetSize > maxPacketSizeInBytes) { + throw new IllegalArgumentException(); + } + ensureOpen(); + int bytesRead = in.readNBytes(packetBuffer, 0, packetSize); + if (bytesRead != packetSize) { + throw new IllegalArgumentException(); + } + if (currentPacketInfo.getAuthenticationTag().length != authenticationTagSizeInBytes) { + throw new IllegalArgumentException(); + } + System.arraycopy(currentPacketInfo.getAuthenticationTag(), 0, packetBuffer, packetSize, authenticationTagSizeInBytes); + Cipher packetCipher = getPacketDecryptionCipher(currentPacketInfo.getIv()); + final int bytesDecrypted; + try { + // in-place decryption + bytesDecrypted = packetCipher.doFinal(packetBuffer, 0, packetSize + authenticationTagSizeInBytes, packetBuffer); + } catch (ShortBufferException | IllegalBlockSizeException | BadPaddingException e) { + throw new IOException(e); + } + if (bytesDecrypted != packetSize) { + throw new IllegalStateException(); + } + return packetSize; + } + + private Cipher getPacketDecryptionCipher(byte[] packetIv) throws IOException { + GCMParameterSpec gcmParameterSpec = new GCMParameterSpec(EncryptedRepository.GCM_TAG_SIZE_IN_BYTES * Byte.SIZE, packetIv); + try { + Cipher packetCipher = Cipher.getInstance(EncryptedRepository.GCM_ENCRYPTION_SCHEME); + packetCipher.init(Cipher.DECRYPT_MODE, secretKey, gcmParameterSpec); + return packetCipher; + } catch (NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException | InvalidAlgorithmParameterException e) { + throw new IOException(e); + } + } + +} diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java index 18aaa5eb7518b..807cb26ff4c61 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java @@ -30,16 +30,11 @@ // not thread-safe public class GCMPacketsEncryptorInputStream extends FilterInputStream { - private static final int GCM_TAG_SIZE_IN_BYTES = 16; - private static final int GCM_IV_SIZE_IN_BYTES = 12; - private static final String GCM_ENCRYPTION_SCHEME = "AES/GCM/NoPadding"; - private static final int AES_BLOCK_SIZE_IN_BYTES = 128; - private final Logger logger = LogManager.getLogger(getClass()); private final int maxPacketSizeInBytes; private final byte[] packetTrailByteBuffer; private final SecretKey secretKey; - private final IvRandomGenerator ivGenerator; + private final IvRandomUniqueGenerator ivGenerator; private final List packetInfoList; private int bytesRemainingInPacket; @@ -51,13 +46,13 @@ public class GCMPacketsEncryptorInputStream extends FilterInputStream { protected GCMPacketsEncryptorInputStream(InputStream in, SecretKey secretKey, int maxPacketSizeInBytes) throws IOException { super(in); this.maxPacketSizeInBytes = maxPacketSizeInBytes; - this.packetTrailByteBuffer = new byte[AES_BLOCK_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES]; + this.packetTrailByteBuffer = new byte[EncryptedRepository.AES_BLOCK_SIZE_IN_BYTES + EncryptedRepository.GCM_TAG_SIZE_IN_BYTES]; this.secretKey = secretKey; - this.ivGenerator = new IvRandomGenerator(); + this.ivGenerator = new IvRandomUniqueGenerator(); this.packetInfoList = new ArrayList<>(); this.bytesRemainingInPacket = maxPacketSizeInBytes; this.packetIv = ivGenerator.newUniqueIv(); - this.packetCipher = getPacketEncryptionCipher(secretKey, packetIv); + this.packetCipher = getPacketEncryptionCipher(packetIv); this.closed = false; this.markPacketIndex = -1; } @@ -86,7 +81,8 @@ public int read(byte b[], int off, int len) throws IOException { } catch (ShortBufferException e) { throw new IllegalStateException(e); } - if (bytesRemainingInPacket == 0 || readSize % AES_BLOCK_SIZE_IN_BYTES != 0) { + if (bytesRemainingInPacket == 0 || readSize % EncryptedRepository.AES_BLOCK_SIZE_IN_BYTES != 0) { + // finalize packet final byte[] authenticationTag; if (encryptedSize == readSize) { try { @@ -95,7 +91,7 @@ public int read(byte b[], int off, int len) throws IOException { throw new IOException(e); } } else { - if (readSize - encryptedSize >= AES_BLOCK_SIZE_IN_BYTES) { + if (readSize - encryptedSize >= EncryptedRepository.AES_BLOCK_SIZE_IN_BYTES) { throw new IllegalStateException(); } int trailAndTagSize = 0; @@ -104,14 +100,14 @@ public int read(byte b[], int off, int len) throws IOException { } catch (IllegalBlockSizeException | ShortBufferException | BadPaddingException e) { throw new IOException(e); } - if (encryptedSize + trailAndTagSize != readSize + GCM_TAG_SIZE_IN_BYTES) { + if (encryptedSize + trailAndTagSize != readSize + EncryptedRepository.GCM_TAG_SIZE_IN_BYTES) { throw new IllegalStateException(); } // copy the remaining packet trail bytes - System.arraycopy(packetTrailByteBuffer, 0, b, off + encryptedSize, trailAndTagSize - GCM_TAG_SIZE_IN_BYTES); - authenticationTag = Arrays.copyOfRange(packetTrailByteBuffer, trailAndTagSize - GCM_TAG_SIZE_IN_BYTES, trailAndTagSize); + System.arraycopy(packetTrailByteBuffer, 0, b, off + encryptedSize, trailAndTagSize - EncryptedRepository.GCM_TAG_SIZE_IN_BYTES); + authenticationTag = Arrays.copyOfRange(packetTrailByteBuffer, trailAndTagSize - EncryptedRepository.GCM_TAG_SIZE_IN_BYTES, trailAndTagSize); } - if (authenticationTag.length != GCM_TAG_SIZE_IN_BYTES) { + if (authenticationTag.length != EncryptedRepository.GCM_TAG_SIZE_IN_BYTES) { throw new IllegalStateException(); } finishPacket(authenticationTag); @@ -145,6 +141,7 @@ public void close() throws IOException { closed = true; in.close(); if (bytesRemainingInPacket < maxPacketSizeInBytes) { + // finish last packet final byte[] authenticationTag; try { authenticationTag = packetCipher.doFinal(); @@ -158,8 +155,8 @@ public void close() throws IOException { @Override public void mark(int readLimit) { in.mark(readLimit); - // finish in-progress packet if (bytesRemainingInPacket < maxPacketSizeInBytes) { + // finish in-progress packet try { byte[] authenticationTag = packetCipher.doFinal(); finishPacket(authenticationTag); @@ -181,7 +178,7 @@ public void reset() throws IOException { // reinstantiate packetCipher bytesRemainingInPacket = maxPacketSizeInBytes; packetIv = ivGenerator.newUniqueIv(); - packetCipher = getPacketEncryptionCipher(secretKey, packetIv); + packetCipher = getPacketEncryptionCipher(packetIv); } public List getEncryptionPacketMetadata() { @@ -202,13 +199,13 @@ private void finishPacket(byte[] authenticationTag) throws IOException { maxPacketSizeInBytes - bytesRemainingInPacket)); bytesRemainingInPacket = maxPacketSizeInBytes; packetIv = ivGenerator.newUniqueIv(); - packetCipher = getPacketEncryptionCipher(secretKey, packetIv); + packetCipher = getPacketEncryptionCipher(packetIv); } - private static Cipher getPacketEncryptionCipher(SecretKey secretKey, byte[] packetIv) throws IOException { - GCMParameterSpec gcmParameterSpec = new GCMParameterSpec(GCM_TAG_SIZE_IN_BYTES * Byte.SIZE, packetIv); + private Cipher getPacketEncryptionCipher(byte[] packetIv) throws IOException { + GCMParameterSpec gcmParameterSpec = new GCMParameterSpec(EncryptedRepository.GCM_TAG_SIZE_IN_BYTES * Byte.SIZE, packetIv); try { - Cipher packetCipher = Cipher.getInstance(GCM_ENCRYPTION_SCHEME); + Cipher packetCipher = Cipher.getInstance(EncryptedRepository.GCM_ENCRYPTION_SCHEME); packetCipher.init(Cipher.ENCRYPT_MODE, secretKey, gcmParameterSpec); return packetCipher; } catch (NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException | InvalidAlgorithmParameterException e) { @@ -218,8 +215,9 @@ private static Cipher getPacketEncryptionCipher(SecretKey secretKey, byte[] pack /** * Tries to return a read size value such that it is smaller or equal to the requested {@code len}, does not exceed the remaining - * space in the current packet and, very important, is a multiple of {@link #AES_BLOCK_SIZE_IN_BYTES}. If the requested {@code len} - * or the remaining space in the current packet are smaller than {@link #AES_BLOCK_SIZE_IN_BYTES}, then their minimum is returned. + * space in the current packet and, very important, is a multiple of {@link EncryptedRepository#AES_BLOCK_SIZE_IN_BYTES}. If the + * requested {@code len} or the remaining space in the current packet are smaller than + * {@link EncryptedRepository#AES_BLOCK_SIZE_IN_BYTES}, then their minimum is returned. * * @param len the requested read size */ @@ -234,23 +232,23 @@ private int getReadSize(int len) { return 0; } int maxReadSize = Math.min(len, bytesRemainingInPacket); - int readSize = (maxReadSize / AES_BLOCK_SIZE_IN_BYTES) * AES_BLOCK_SIZE_IN_BYTES; + int readSize = (maxReadSize / EncryptedRepository.AES_BLOCK_SIZE_IN_BYTES) * EncryptedRepository.AES_BLOCK_SIZE_IN_BYTES; if (readSize != 0) { return readSize; } - assert maxReadSize < AES_BLOCK_SIZE_IN_BYTES; + assert maxReadSize < EncryptedRepository.AES_BLOCK_SIZE_IN_BYTES; if (maxReadSize == len) { - logger.warn("Reading [" + len + "] bytes, which is less than [" + AES_BLOCK_SIZE_IN_BYTES + "], is terribly inefficient."); + logger.warn("Reading [" + len + "] bytes, which is less than [" + EncryptedRepository.AES_BLOCK_SIZE_IN_BYTES + "], is terribly inefficient."); } return maxReadSize; } - static class IvRandomGenerator { + static class IvRandomUniqueGenerator { private final Map> generatedIvs; private final SecureRandom secureRandom; - IvRandomGenerator() { + IvRandomUniqueGenerator() { generatedIvs = new HashMap<>(); secureRandom = new SecureRandom(); } @@ -269,7 +267,7 @@ private byte[] newUniqueIv(int retryCount) { if (false == part2Set.add(part2)) { return newUniqueIv(retryCount - 1); } - ByteBuffer uniqueIv = ByteBuffer.allocate(GCM_IV_SIZE_IN_BYTES); + ByteBuffer uniqueIv = ByteBuffer.allocate(EncryptedRepository.GCM_IV_SIZE_IN_BYTES); uniqueIv.putLong(part1); uniqueIv.putInt(part2); return uniqueIv.array(); From a460b0956d777e83ae6460be0aba5280bcfa40ea Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 21 Nov 2019 14:19:56 +0200 Subject: [PATCH 24/29] Before EncryptedRepository retrofitting --- .../encrypted/BlobEncryptionMetadata.java | 215 ++++++++++++++++++ .../encrypted/EncryptedBlobMetadata.java | 181 --------------- .../GCMPacketsDecryptorInputStream.java | 18 +- .../GCMPacketsEncryptorInputStream.java | 9 +- 4 files changed, 228 insertions(+), 195 deletions(-) create mode 100644 x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/BlobEncryptionMetadata.java delete mode 100644 x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedBlobMetadata.java diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/BlobEncryptionMetadata.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/BlobEncryptionMetadata.java new file mode 100644 index 0000000000000..dc091d8ea9868 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/BlobEncryptionMetadata.java @@ -0,0 +1,215 @@ +package org.elasticsearch.repositories.encrypted; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.SequenceInputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Enumeration; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; + +public class BlobEncryptionMetadata { + + private static final int FIELDS_BUFFER_SIZE = 5 * Integer.SIZE / Byte.SIZE; + + private final int maxPacketSizeInBytes; + private final int authTagSizeInBytes; + private final int ivSizeInBytes; + private final byte[] dataEncryptionKeyMaterial; + private List packetsInfoList; + + public BlobEncryptionMetadata(int maxPacketSizeInBytes, int ivSizeInBytes, int authTagSizeInBytes, + byte[] dataEncryptionKeyMaterial, List packetsInfoList) { + this.maxPacketSizeInBytes = maxPacketSizeInBytes; + this.ivSizeInBytes = ivSizeInBytes; + this.authTagSizeInBytes = authTagSizeInBytes; + this.dataEncryptionKeyMaterial = dataEncryptionKeyMaterial; + // consistency check of the packet infos + for (PacketInfo packetInfo : packetsInfoList) { + if (packetInfo.getSizeInBytes() > maxPacketSizeInBytes) { + throw new IllegalArgumentException(); + } + if (packetInfo.getIv().length != ivSizeInBytes) { + throw new IllegalArgumentException(); + } + if (packetInfo.getAuthTag().length != authTagSizeInBytes) { + throw new IllegalArgumentException(); + } + } + this.packetsInfoList = Collections.unmodifiableList(packetsInfoList); + } + + public BlobEncryptionMetadata(InputStream inputStream) throws IOException { + ByteBuffer fieldsByteBuffer = ByteBuffer.allocate(FIELDS_BUFFER_SIZE).order(ByteOrder.BIG_ENDIAN); + if (FIELDS_BUFFER_SIZE != inputStream.readNBytes(fieldsByteBuffer.array(), 0, FIELDS_BUFFER_SIZE)) { + throw new IllegalArgumentException(); + } + this.maxPacketSizeInBytes = fieldsByteBuffer.getInt(); + this.authTagSizeInBytes = fieldsByteBuffer.getInt(); + this.ivSizeInBytes = fieldsByteBuffer.getInt(); + int dataEncryptionKeySizeInBytes = fieldsByteBuffer.getInt(); + int packetsInfoListSize = fieldsByteBuffer.getInt(); + this.dataEncryptionKeyMaterial = new byte[dataEncryptionKeySizeInBytes]; + if (dataEncryptionKeySizeInBytes != inputStream.readNBytes(dataEncryptionKeyMaterial, 0, dataEncryptionKeySizeInBytes)) { + throw new IllegalArgumentException(); + } + List packetsInfo = new ArrayList<>(packetsInfoListSize); + for (int i = 0; i < packetsInfoListSize; i++) { + PacketInfo packetInfo = new PacketInfo(inputStream, ivSizeInBytes, authTagSizeInBytes); + // consistency check of the packet infos + if (packetInfo.getSizeInBytes() > this.maxPacketSizeInBytes) { + throw new IllegalArgumentException(); + } + if (packetInfo.getIv().length != this.ivSizeInBytes) { + throw new IllegalArgumentException(); + } + if (packetInfo.getAuthTag().length != this.authTagSizeInBytes) { + throw new IllegalArgumentException(); + } + packetsInfo.add(packetInfo); + } + this.packetsInfoList = Collections.unmodifiableList(packetsInfo); + } + + public byte[] getDataEncryptionKeyMaterial() { + return dataEncryptionKeyMaterial; + } + + public int getMaxPacketSizeInBytes() { + return maxPacketSizeInBytes; + } + + public int getAuthTagSizeInBytes() { + return authTagSizeInBytes; + } + + public int getIvSizeInBytes() { + return ivSizeInBytes; + } + + public List getPacketsInfoList() { + return packetsInfoList; + } + + public InputStream toInputStream() { + + final ByteBuffer fieldsByteBuffer = ByteBuffer.allocate(FIELDS_BUFFER_SIZE).order(ByteOrder.BIG_ENDIAN); + fieldsByteBuffer.putInt(maxPacketSizeInBytes); + fieldsByteBuffer.putInt(authTagSizeInBytes); + fieldsByteBuffer.putInt(ivSizeInBytes); + fieldsByteBuffer.putInt(dataEncryptionKeyMaterial.length); + fieldsByteBuffer.putInt(packetsInfoList.size()); + + return new SequenceInputStream(new ByteArrayInputStream(fieldsByteBuffer.array()), + new SequenceInputStream(new ByteArrayInputStream(dataEncryptionKeyMaterial), + new SequenceInputStream(new Enumeration() { + + private final Iterator packetInfoIterator = packetsInfoList.iterator(); + + @Override + public boolean hasMoreElements() { + return packetInfoIterator.hasNext(); + } + + @Override + public InputStream nextElement() { + return packetInfoIterator.next().toInputStream(); + } + }))); + } + + static class PacketInfo { + + private final byte[] iv; + private final byte[] authTag; + private final int sizeInBytes; + + PacketInfo(byte[] iv, byte[] authTag, int sizeInBytes) { + this.iv = iv; + this.authTag = authTag; + this.sizeInBytes = sizeInBytes; + } + + PacketInfo(InputStream inputStream, int ivSizeInBytes, int authTagSizeInBytes) throws IOException { + this.iv = new byte[ivSizeInBytes]; + if (ivSizeInBytes != inputStream.readNBytes(iv, 0, ivSizeInBytes)) { + throw new IllegalArgumentException(); + } + this.authTag = new byte[authTagSizeInBytes]; + if (authTagSizeInBytes != inputStream.readNBytes(authTag, 0, authTagSizeInBytes)) { + throw new IllegalArgumentException(); + } + this.sizeInBytes = ((inputStream.read() & 0xFF) << 24) | + ((inputStream.read() & 0xFF) << 16) | + ((inputStream.read() & 0xFF) << 8 ) | + ((inputStream.read() & 0xFF) << 0 ); + } + + byte[] getIv() { + return iv; + } + + byte[] getAuthTag() { + return authTag; + } + + int getSizeInBytes() { + return sizeInBytes; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PacketInfo that = (PacketInfo) o; + return sizeInBytes == that.sizeInBytes && + Arrays.equals(iv, that.iv) && + Arrays.equals(authTag, that.authTag); + } + + @Override + public int hashCode() { + int result = Objects.hash(sizeInBytes); + result = 31 * result + Arrays.hashCode(iv); + result = 31 * result + Arrays.hashCode(authTag); + return result; + } + + InputStream toInputStream() { + return new InputStream() { + + private int idx = 0; + + @Override + public int read() throws IOException { + if (idx < iv.length) { + return iv[idx++]; + } else if (idx < iv.length + authTag.length) { + return authTag[idx++ - iv.length]; + } else if (idx < iv.length + authTag.length + 4) { + idx++; + if (idx == iv.length + authTag.length + 1) { + return (byte) (sizeInBytes >>> 24); + } else if (idx == iv.length + authTag.length + 2) { + return (byte) (sizeInBytes >>> 16); + } else if (idx == iv.length + authTag.length + 3) { + return (byte) (sizeInBytes >>> 8); + } else { + return (byte) sizeInBytes; + } + } else { + return -1; + } + } + }; + } + + } + +} diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedBlobMetadata.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedBlobMetadata.java deleted file mode 100644 index 669df5a181599..0000000000000 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedBlobMetadata.java +++ /dev/null @@ -1,181 +0,0 @@ -package org.elasticsearch.repositories.encrypted; - -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.ToXContentObject; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Base64; -import java.util.Collections; -import java.util.List; -import java.util.Objects; - -import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; - -public class EncryptedBlobMetadata implements ToXContentObject { - - private static ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("encrypted_blob_metadata", false, - args -> { - @SuppressWarnings("unchecked") Integer version = (Integer) args[0]; - @SuppressWarnings("unchecked") String wrappedDataEncryptionKey = (String) args[1]; - @SuppressWarnings("unchecked") Integer maximumPacketSizeInBytes = (Integer) args[2]; - @SuppressWarnings("unchecked") Integer authenticationTagSizeInBytes = (Integer) args[3]; - @SuppressWarnings("unchecked") List packetsInfo = (List) args[4]; - return new EncryptedBlobMetadata(version, Base64.getDecoder().decode(wrappedDataEncryptionKey), - maximumPacketSizeInBytes, authenticationTagSizeInBytes, packetsInfo); - } - ); - - static { - PARSER.declareInt(constructorArg(), Fields.VERSION); - PARSER.declareString(constructorArg(), Fields.WRAPPED_DEK); - PARSER.declareInt(constructorArg(), Fields.MAXIMUM_PACKET_SIZE); - PARSER.declareInt(constructorArg(), Fields.AUTH_TAG_SIZE); - PARSER.declareObjectArray(constructorArg(), (p, c) -> PacketInfo.fromXContent(p), Fields.PACKETS_INFO); - } - - public static EncryptedBlobMetadata fromXContent(XContentParser parser) { - return PARSER.apply(parser, null); - } - - private final int version; - private final byte[] wrappedDataEncryptionKey; - private final int maximumPacketSizeInBytes; - private final int authenticationTagSizeInBytes; - private List packetsInfo; - - public EncryptedBlobMetadata(int version, byte[] wrappedDataEncryptionKey, int maximumPacketSizeInBytes, - int authenticationTagSizeInBytes, List packetsInfo) { - this.version = version; - this.wrappedDataEncryptionKey = wrappedDataEncryptionKey; - this.maximumPacketSizeInBytes = maximumPacketSizeInBytes; - this.authenticationTagSizeInBytes = authenticationTagSizeInBytes; - this.packetsInfo = Collections.unmodifiableList(packetsInfo); - } - - public byte[] getWrappedDataEncryptionKey() { - return wrappedDataEncryptionKey; - } - - public int getVersion() { - return version; - } - - public int getMaximumPacketSizeInBytes() { - return maximumPacketSizeInBytes; - } - - public List getPacketsInfo() { - return packetsInfo; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field(Fields.VERSION.getPreferredName(), version); - builder.field(Fields.WRAPPED_DEK.getPreferredName(), Base64.getEncoder().encodeToString(wrappedDataEncryptionKey)); - builder.field(Fields.MAXIMUM_PACKET_SIZE.getPreferredName(), maximumPacketSizeInBytes); - builder.field(Fields.AUTH_TAG_SIZE.getPreferredName(), authenticationTagSizeInBytes); - builder.startArray(Fields.PACKETS_INFO.getPreferredName()); - for (PacketInfo packetInfo : packetsInfo) { - packetInfo.toXContent(builder, ToXContent.EMPTY_PARAMS); - } - builder.endArray(); - builder.endObject(); - return null; - } - - private interface Fields { - ParseField VERSION = new ParseField("version"); - ParseField WRAPPED_DEK = new ParseField("wrappedDataEncryptionKey"); - ParseField MAXIMUM_PACKET_SIZE = new ParseField("maximumPacketSizeInBytes"); - ParseField AUTH_TAG_SIZE = new ParseField("authenticationTagSizeInBytes"); - ParseField PACKETS_INFO = new ParseField("packetsInfo"); - } - - static class PacketInfo implements ToXContentObject { - - private static ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("packet_info", false, - args -> { - @SuppressWarnings("unchecked") String iv = (String) args[0]; - @SuppressWarnings("unchecked") String authenticationTag = (String) args[1]; - @SuppressWarnings("unchecked") Integer sizeInBytes = (Integer) args[2]; - return new PacketInfo(Base64.getDecoder().decode(iv), Base64.getDecoder().decode(authenticationTag), - sizeInBytes); - } - ); - - static { - PARSER.declareString(constructorArg(), Fields.IV); - PARSER.declareString(constructorArg(), Fields.AUTHENTICATION_TAG); - PARSER.declareInt(constructorArg(), Fields.SIZE_IN_BYTES); - } - - static PacketInfo fromXContent(XContentParser parser) { - return PARSER.apply(parser, null); - } - - private final byte[] iv; - private final byte[] authenticationTag; - private final int sizeInBytes; - - PacketInfo(byte[] iv, byte[] authenticationTag, int sizeInBytes) { - this.iv = iv; - this.authenticationTag = authenticationTag; - this.sizeInBytes = sizeInBytes; - } - - byte[] getIv() { - return iv; - } - - byte[] getAuthenticationTag() { - return authenticationTag; - } - - int getSizeInBytes() { - return sizeInBytes; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - PacketInfo that = (PacketInfo) o; - return sizeInBytes == that.sizeInBytes && - Arrays.equals(iv, that.iv) && - Arrays.equals(authenticationTag, that.authenticationTag); - } - - @Override - public int hashCode() { - int result = Objects.hash(sizeInBytes); - result = 31 * result + Arrays.hashCode(iv); - result = 31 * result + Arrays.hashCode(authenticationTag); - return result; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field(Fields.IV.getPreferredName(), Base64.getEncoder().encodeToString(iv)); - builder.field(Fields.AUTHENTICATION_TAG.getPreferredName(), Base64.getEncoder().encodeToString(authenticationTag)); - builder.field(Fields.SIZE_IN_BYTES.getPreferredName(), sizeInBytes); - builder.endObject(); - return null; - } - - private interface Fields { - ParseField IV = new ParseField("iv"); - ParseField AUTHENTICATION_TAG = new ParseField("authenticationTag"); - ParseField SIZE_IN_BYTES = new ParseField("sizeInBytes"); - } - } - -} diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsDecryptorInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsDecryptorInputStream.java index f9d513ddb5f49..884622d0ab80f 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsDecryptorInputStream.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsDecryptorInputStream.java @@ -13,14 +13,14 @@ import java.security.InvalidAlgorithmParameterException; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; -import java.util.List; +import java.util.Iterator; public class GCMPacketsDecryptorInputStream extends FilterInputStream { private final int maxPacketSizeInBytes; private final int authenticationTagSizeInBytes; private final SecretKey secretKey; - private final List packetInfoList; + private final Iterator packetInfoIterator; private final byte[] packetBuffer; private int packetIndex; @@ -29,14 +29,14 @@ public class GCMPacketsDecryptorInputStream extends FilterInputStream { private boolean closed; protected GCMPacketsDecryptorInputStream(InputStream in, SecretKey secretKey, int maxPacketSizeInBytes, - int authenticationTagSizeInBytes, List packetInfoList) { + int authenticationTagSizeInBytes, + Iterator packetInfoIterator) { super(in); this.secretKey = secretKey; this.maxPacketSizeInBytes = maxPacketSizeInBytes; this.authenticationTagSizeInBytes = authenticationTagSizeInBytes; - this.packetInfoList = packetInfoList; + this.packetInfoIterator = packetInfoIterator; this.packetBuffer = new byte[maxPacketSizeInBytes + authenticationTagSizeInBytes]; - this.packetIndex = 0; this.bufferStartOffset = 0; this.bufferEndOffset = 0; this.closed = false; @@ -127,10 +127,10 @@ void ensureOpen() throws IOException { } private int readAndDecryptNextPacket() throws IOException { - if (packetIndex == packetInfoList.size()) { + if (false == packetInfoIterator.hasNext()) { return -1; } - EncryptedBlobMetadata.PacketInfo currentPacketInfo = packetInfoList.get(packetIndex++); + BlobEncryptionMetadata.PacketInfo currentPacketInfo = packetInfoIterator.next(); int packetSize = currentPacketInfo.getSizeInBytes(); if (packetSize > maxPacketSizeInBytes) { throw new IllegalArgumentException(); @@ -140,10 +140,10 @@ private int readAndDecryptNextPacket() throws IOException { if (bytesRead != packetSize) { throw new IllegalArgumentException(); } - if (currentPacketInfo.getAuthenticationTag().length != authenticationTagSizeInBytes) { + if (currentPacketInfo.getAuthTag().length != authenticationTagSizeInBytes) { throw new IllegalArgumentException(); } - System.arraycopy(currentPacketInfo.getAuthenticationTag(), 0, packetBuffer, packetSize, authenticationTagSizeInBytes); + System.arraycopy(currentPacketInfo.getAuthTag(), 0, packetBuffer, packetSize, authenticationTagSizeInBytes); Cipher packetCipher = getPacketDecryptionCipher(currentPacketInfo.getIv()); final int bytesDecrypted; try { diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java index 807cb26ff4c61..079991ce933e0 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java @@ -35,7 +35,7 @@ public class GCMPacketsEncryptorInputStream extends FilterInputStream { private final byte[] packetTrailByteBuffer; private final SecretKey secretKey; private final IvRandomUniqueGenerator ivGenerator; - private final List packetInfoList; + private final List packetInfoList; private int bytesRemainingInPacket; private byte[] packetIv; @@ -181,7 +181,7 @@ public void reset() throws IOException { packetCipher = getPacketEncryptionCipher(packetIv); } - public List getEncryptionPacketMetadata() { + public List getEncryptionPacketMetadata() { if (false == closed) { throw new IllegalStateException(); } @@ -194,9 +194,8 @@ void ensureOpen() throws IOException { } } - private void finishPacket(byte[] authenticationTag) throws IOException { - packetInfoList.add(new EncryptedBlobMetadata.PacketInfo(packetIv, authenticationTag, - maxPacketSizeInBytes - bytesRemainingInPacket)); + private void finishPacket(byte[] authTag) throws IOException { + packetInfoList.add(new BlobEncryptionMetadata.PacketInfo(packetIv, authTag, maxPacketSizeInBytes - bytesRemainingInPacket)); bytesRemainingInPacket = maxPacketSizeInBytes; packetIv = ivGenerator.newUniqueIv(); packetCipher = getPacketEncryptionCipher(packetIv); From 835c9d29a02d708669ddd9aa3e045f7c637e616e Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 21 Nov 2019 20:10:50 +0200 Subject: [PATCH 25/29] BlobEncriptionMetadata no InputStream use write method --- .../encrypted/BlobEncryptionMetadata.java | 125 +++++++----------- 1 file changed, 45 insertions(+), 80 deletions(-) diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/BlobEncryptionMetadata.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/BlobEncryptionMetadata.java index dc091d8ea9868..4dda3bff48f05 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/BlobEncryptionMetadata.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/BlobEncryptionMetadata.java @@ -1,23 +1,16 @@ package org.elasticsearch.repositories.encrypted; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.io.SequenceInputStream; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; +import java.io.OutputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Enumeration; -import java.util.Iterator; import java.util.List; import java.util.Objects; public class BlobEncryptionMetadata { - private static final int FIELDS_BUFFER_SIZE = 5 * Integer.SIZE / Byte.SIZE; - private final int maxPacketSizeInBytes; private final int authTagSizeInBytes; private final int ivSizeInBytes; @@ -46,19 +39,14 @@ public BlobEncryptionMetadata(int maxPacketSizeInBytes, int ivSizeInBytes, int a } public BlobEncryptionMetadata(InputStream inputStream) throws IOException { - ByteBuffer fieldsByteBuffer = ByteBuffer.allocate(FIELDS_BUFFER_SIZE).order(ByteOrder.BIG_ENDIAN); - if (FIELDS_BUFFER_SIZE != inputStream.readNBytes(fieldsByteBuffer.array(), 0, FIELDS_BUFFER_SIZE)) { - throw new IllegalArgumentException(); - } - this.maxPacketSizeInBytes = fieldsByteBuffer.getInt(); - this.authTagSizeInBytes = fieldsByteBuffer.getInt(); - this.ivSizeInBytes = fieldsByteBuffer.getInt(); - int dataEncryptionKeySizeInBytes = fieldsByteBuffer.getInt(); - int packetsInfoListSize = fieldsByteBuffer.getInt(); - this.dataEncryptionKeyMaterial = new byte[dataEncryptionKeySizeInBytes]; - if (dataEncryptionKeySizeInBytes != inputStream.readNBytes(dataEncryptionKeyMaterial, 0, dataEncryptionKeySizeInBytes)) { - throw new IllegalArgumentException(); - } + this.maxPacketSizeInBytes = readInt(inputStream); + this.authTagSizeInBytes = readInt(inputStream); + this.ivSizeInBytes = readInt(inputStream); + + int dataEncryptionKeySizeInBytes = readInt(inputStream); + this.dataEncryptionKeyMaterial = readExactlyNBytes(inputStream, dataEncryptionKeySizeInBytes); + + int packetsInfoListSize = readInt(inputStream); List packetsInfo = new ArrayList<>(packetsInfoListSize); for (int i = 0; i < packetsInfoListSize; i++) { PacketInfo packetInfo = new PacketInfo(inputStream, ivSizeInBytes, authTagSizeInBytes); @@ -97,31 +85,40 @@ public List getPacketsInfoList() { return packetsInfoList; } - public InputStream toInputStream() { + public void write(OutputStream out) throws IOException { + writeInt(out, maxPacketSizeInBytes); + writeInt(out, authTagSizeInBytes); + writeInt(out, ivSizeInBytes); - final ByteBuffer fieldsByteBuffer = ByteBuffer.allocate(FIELDS_BUFFER_SIZE).order(ByteOrder.BIG_ENDIAN); - fieldsByteBuffer.putInt(maxPacketSizeInBytes); - fieldsByteBuffer.putInt(authTagSizeInBytes); - fieldsByteBuffer.putInt(ivSizeInBytes); - fieldsByteBuffer.putInt(dataEncryptionKeyMaterial.length); - fieldsByteBuffer.putInt(packetsInfoList.size()); + writeInt(out, dataEncryptionKeyMaterial.length); + out.write(dataEncryptionKeyMaterial); - return new SequenceInputStream(new ByteArrayInputStream(fieldsByteBuffer.array()), - new SequenceInputStream(new ByteArrayInputStream(dataEncryptionKeyMaterial), - new SequenceInputStream(new Enumeration() { + writeInt(out, packetsInfoList.size()); + for (PacketInfo packetInfo : packetsInfoList) { + packetInfo.write(out); + } + } - private final Iterator packetInfoIterator = packetsInfoList.iterator(); + private static int readInt(InputStream inputStream) throws IOException { + return ((inputStream.read() & 0xFF) << 24) | + ((inputStream.read() & 0xFF) << 16) | + ((inputStream.read() & 0xFF) << 8 ) | + ((inputStream.read() & 0xFF) << 0 ); + } - @Override - public boolean hasMoreElements() { - return packetInfoIterator.hasNext(); - } + private static void writeInt(OutputStream out, int val) throws IOException { + out.write(val >>> 24); + out.write(val >>> 16); + out.write(val >>> 8); + out.write(val); + } - @Override - public InputStream nextElement() { - return packetInfoIterator.next().toInputStream(); - } - }))); + private static byte[] readExactlyNBytes(InputStream inputStream, int nBytes) throws IOException { + byte[] ans = new byte[nBytes]; + if (nBytes != inputStream.readNBytes(ans, 0, nBytes)) { + throw new IOException("Fewer than [" + nBytes + "] read"); + } + return ans; } static class PacketInfo { @@ -137,18 +134,9 @@ static class PacketInfo { } PacketInfo(InputStream inputStream, int ivSizeInBytes, int authTagSizeInBytes) throws IOException { - this.iv = new byte[ivSizeInBytes]; - if (ivSizeInBytes != inputStream.readNBytes(iv, 0, ivSizeInBytes)) { - throw new IllegalArgumentException(); - } - this.authTag = new byte[authTagSizeInBytes]; - if (authTagSizeInBytes != inputStream.readNBytes(authTag, 0, authTagSizeInBytes)) { - throw new IllegalArgumentException(); - } - this.sizeInBytes = ((inputStream.read() & 0xFF) << 24) | - ((inputStream.read() & 0xFF) << 16) | - ((inputStream.read() & 0xFF) << 8 ) | - ((inputStream.read() & 0xFF) << 0 ); + this.iv = readExactlyNBytes(inputStream, ivSizeInBytes); + this.authTag = readExactlyNBytes(inputStream, authTagSizeInBytes); + this.sizeInBytes = readInt(inputStream); } byte[] getIv() { @@ -181,33 +169,10 @@ public int hashCode() { return result; } - InputStream toInputStream() { - return new InputStream() { - - private int idx = 0; - - @Override - public int read() throws IOException { - if (idx < iv.length) { - return iv[idx++]; - } else if (idx < iv.length + authTag.length) { - return authTag[idx++ - iv.length]; - } else if (idx < iv.length + authTag.length + 4) { - idx++; - if (idx == iv.length + authTag.length + 1) { - return (byte) (sizeInBytes >>> 24); - } else if (idx == iv.length + authTag.length + 2) { - return (byte) (sizeInBytes >>> 16); - } else if (idx == iv.length + authTag.length + 3) { - return (byte) (sizeInBytes >>> 8); - } else { - return (byte) sizeInBytes; - } - } else { - return -1; - } - } - }; + public void write(OutputStream out) throws IOException { + out.write(iv); + out.write(authTag); + writeInt(out, sizeInBytes); } } From dd7f4e13e8a7d59aac7655ad884dbd4c1bcc63ca Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Fri, 22 Nov 2019 00:17:02 +0200 Subject: [PATCH 26/29] All crypto in-place! --- .../encrypted/EncryptedRepository.java | 220 +++++++++--------- .../GCMPacketsEncryptorInputStream.java | 5 +- 2 files changed, 115 insertions(+), 110 deletions(-) diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java index ab171e234e77b..3400f7a169ee4 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java @@ -6,8 +6,20 @@ package org.elasticsearch.repositories.encrypted; -import org.bouncycastle.crypto.fips.FipsUnapprovedOperationError; -import org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider; +import org.bouncycastle.asn1.ASN1ObjectIdentifier; +import org.bouncycastle.asn1.cms.CMSObjectIdentifiers; +import org.bouncycastle.cms.CMSAlgorithm; +import org.bouncycastle.cms.CMSEnvelopedData; +import org.bouncycastle.cms.CMSEnvelopedDataGenerator; +import org.bouncycastle.cms.CMSException; +import org.bouncycastle.cms.CMSTypedData; +import org.bouncycastle.cms.PasswordRecipientInfoGenerator; +import org.bouncycastle.cms.PasswordRecipientInformation; +import org.bouncycastle.cms.RecipientInformation; +import org.bouncycastle.cms.RecipientInformationStore; +import org.bouncycastle.cms.jcajce.JceCMSContentEncryptorBuilder; +import org.bouncycastle.cms.jcajce.JcePasswordEnvelopedRecipient; +import org.bouncycastle.cms.jcajce.JcePasswordRecipientInfoGenerator; import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.Strings; import org.elasticsearch.common.blobstore.BlobContainer; @@ -16,7 +28,6 @@ import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.blobstore.DeleteResult; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.hash.MessageDigests; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.SecureSetting; import org.elasticsearch.common.settings.SecureString; @@ -25,45 +36,32 @@ import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; -import javax.crypto.Cipher; -import javax.crypto.IllegalBlockSizeException; import javax.crypto.KeyGenerator; -import javax.crypto.NoSuchPaddingException; import javax.crypto.SecretKey; -import javax.crypto.SecretKeyFactory; -import javax.crypto.spec.PBEKeySpec; import javax.crypto.spec.SecretKeySpec; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.security.AccessController; -import java.security.InvalidKeyException; +import java.io.OutputStream; import java.security.NoSuchAlgorithmException; -import java.security.PrivilegedAction; import java.security.SecureRandom; -import java.security.spec.InvalidKeySpecException; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.function.Function; public class EncryptedRepository extends BlobStoreRepository { + static final int ENCRYPTION_PROTOCOL_VERSION_NUMBER = 1; + // this is chosen somewhat arbitrarily. Assuming no mark calls during encryption/upload/snapshot, a larger value increases + // the metadata/data size ratio, thereby reducing the encryption overhead, but it also requires that decryption/download/restore + // allocate and use a buffer of this size. + static final int MAX_PACKET_SIZE = 64 * 1024; // 64KB packet sizes static final int GCM_TAG_SIZE_IN_BYTES = 16; static final int GCM_IV_SIZE_IN_BYTES = 12; static final String GCM_ENCRYPTION_SCHEME = "AES/GCM/NoPadding"; static final int AES_BLOCK_SIZE_IN_BYTES = 128; - private static BouncyCastleFipsProvider BC_FIPS_PROVIDER; - static { - AccessController.doPrivileged((PrivilegedAction) () -> { - EncryptedRepository.BC_FIPS_PROVIDER = new BouncyCastleFipsProvider(); - return null; - }); - } - // TODO change this and make it (secure) randomly generated for each blob - private static final int NONCE = 5318008; - private static final int ENCRYPTION_PROTOCOL_VERSION_NUMBER = 1; - static final Setting.AffixSetting ENCRYPTION_PASSWORD_SETTING = Setting.affixKeySetting("repository.encrypted.", "password", key -> SecureSetting.secureString(key, null)); @@ -71,17 +69,17 @@ public class EncryptedRepository extends BlobStoreRepository { private static final String ENCRYPTION_METADATA_PREFIX = "encryption-metadata-"; private final BlobStoreRepository delegatedRepository; - private final SecretKey masterSecretKey; + private final char[] masterPassword; - protected EncryptedRepository(BlobStoreRepository delegatedRepository, SecretKey masterSecretKey) { + protected EncryptedRepository(BlobStoreRepository delegatedRepository, char[] masterPassword) { super(delegatedRepository); this.delegatedRepository = delegatedRepository; - this.masterSecretKey = masterSecretKey; + this.masterPassword = masterPassword; } @Override protected BlobStore createBlobStore() throws Exception { - return new EncryptedBlobStoreDecorator(this.delegatedRepository.blobStore(), this.masterSecretKey); + return new EncryptedBlobStoreDecorator(this.delegatedRepository.blobStore(), masterPassword); } @Override @@ -131,14 +129,14 @@ public Repository create(RepositoryMetaData metaData, Function packetsMetadataList = encryptedInputStream.getEncryptionPacketMetadata(); + BlobEncryptionMetadata metadata = new BlobEncryptionMetadata(MAX_PACKET_SIZE, GCM_IV_SIZE_IN_BYTES, GCM_TAG_SIZE_IN_BYTES, + dataEncryptionKey.getEncoded(), packetsMetadataList); + byte[] encryptedMetadata = encryptMetadata(metadata); + try (InputStream stream = new ByteArrayInputStream(encryptedMetadata)) { + this.encryptionMetadataBlobContainer.writeBlob(blobName, stream, encryptedMetadata.length, false); } } + private byte[] encryptMetadata(BlobEncryptionMetadata metadata) throws IOException { + CMSEnvelopedDataGenerator envelopedDataGenerator = new CMSEnvelopedDataGenerator(); + PasswordRecipientInfoGenerator passwordRecipientInfoGenerator = new JcePasswordRecipientInfoGenerator(CMSAlgorithm.AES256_GCM + , masterPassword); + envelopedDataGenerator.addRecipientInfoGenerator(passwordRecipientInfoGenerator); + final CMSEnvelopedData envelopedData; + try { + envelopedData = envelopedDataGenerator.generate(new CMSTypedData() { + @Override + public ASN1ObjectIdentifier getContentType() { + return CMSObjectIdentifiers.data; + } + + @Override + public void write(OutputStream out) throws IOException, CMSException { + metadata.write(out); + } + + @Override + public Object getContent() { + return metadata; + } + }, new JceCMSContentEncryptorBuilder(CMSAlgorithm.AES256_GCM).build()); + } catch (CMSException e) { + throw new IOException(e); + } + return envelopedData.getEncoded(); + } + + private BlobEncryptionMetadata decryptMetadata(byte[] metadata) throws IOException { + final CMSEnvelopedData envelopedData; + try { + envelopedData = new CMSEnvelopedData(metadata); + } catch (CMSException e) { + throw new IOException(e); + } + RecipientInformationStore recipients = envelopedData.getRecipientInfos(); + if (recipients.getRecipients().size() != 1) { + throw new IllegalStateException(); + } + RecipientInformation recipient = recipients.iterator().next(); + if (false == (recipient instanceof PasswordRecipientInformation)) { + throw new IllegalStateException(); + } + final byte[] decryptedMetadata; + try { + decryptedMetadata = recipient.getContent(new JcePasswordEnvelopedRecipient(masterPassword)); + } catch (CMSException e) { + throw new IOException(e); + } + return new BlobEncryptionMetadata(new ByteArrayInputStream(decryptedMetadata)); + } + @Override public void writeBlobAtomic(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) throws IOException { @@ -230,15 +282,14 @@ public void writeBlobAtomic(String blobName, InputStream inputStream, long blobS @Override public void deleteBlob(String blobName) throws IOException { - this.delegatedBlobContainer.deleteBlob(blobName); this.encryptionMetadataBlobContainer.deleteBlob(blobName); + this.delegatedBlobContainer.deleteBlob(blobName); } @Override public DeleteResult delete() throws IOException { - DeleteResult result = this.delegatedBlobContainer.delete(); this.encryptionMetadataBlobContainer.delete(); - return result; + return this.delegatedBlobContainer.delete(); } @Override @@ -253,61 +304,14 @@ public Map children() throws IOException { @Override public Map listBlobsByPrefix(String blobNamePrefix) throws IOException { - Map delegatedBlobs = this.delegatedBlobContainer.listBlobsByPrefix(blobNamePrefix); - Map delegatedBlobsWithPlainSize = new HashMap<>(delegatedBlobs.size()); - for (Map.Entry entry : delegatedBlobs.entrySet()) { - delegatedBlobsWithPlainSize.put(entry.getKey(), new BlobMetaData() { - - @Override - public String name() { - return entry.getValue().name(); - } - - @Override - public long length() { - return GCMPacketsCipherInputStream.getDecryptionSizeFromCipherSize(entry.getValue().length()); - } - }); - } - return delegatedBlobsWithPlainSize; - } - } - - private static SecretKey generateSecretKeyFromPassword(char[] password) throws NoSuchAlgorithmException, InvalidKeySpecException { - byte[] salt = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; // same salt for 1:1 password to key - PBEKeySpec spec = new PBEKeySpec(password, salt, 65536, 256); - try { - SecretKey tmp = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA256", BC_FIPS_PROVIDER).generateSecret(spec); - return new SecretKeySpec(tmp.getEncoded(), "AES"); - } catch (FipsUnapprovedOperationError e) { - // password must be at least 112 bits - // wrap the error into an exception because the exception is better handled up the stack - // TODO don't do this wrapping - throw new RuntimeException(e); + return this.delegatedBlobContainer.listBlobsByPrefix(blobNamePrefix); } } - private static String keyId(SecretKey secretKey) { - return MessageDigests.toHexString(MessageDigests.sha256().digest(secretKey.getEncoded())); - } - private static SecretKey generateRandomSecretKey() throws NoSuchAlgorithmException { - KeyGenerator keyGen = KeyGenerator.getInstance("AES", BC_FIPS_PROVIDER); - keyGen.init(256, SecureRandom.getInstance("DEFAULT", BC_FIPS_PROVIDER)); + KeyGenerator keyGen = KeyGenerator.getInstance("AES"); + keyGen.init(256, SecureRandom.getInstance("DEFAULT")); return keyGen.generateKey(); } - private static byte[] wrapKey(SecretKey toWrap, SecretKey keyWrappingKey) - throws NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException { - Cipher cipher = Cipher.getInstance("AESWrap", BC_FIPS_PROVIDER); - cipher.init(Cipher.WRAP_MODE, keyWrappingKey); - return cipher.wrap(toWrap); - } - - private static SecretKey unwrapKey(byte[] toUnwrap, SecretKey keyEncryptionKey) - throws NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException { - Cipher cipher = Cipher.getInstance("AESWrap", BC_FIPS_PROVIDER); - cipher.init(Cipher.UNWRAP_MODE, keyEncryptionKey); - return (SecretKey) cipher.unwrap(toUnwrap, "AES", Cipher.SECRET_KEY); - } } diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java index 079991ce933e0..abf3a56b8f3fb 100644 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java @@ -21,6 +21,7 @@ import java.security.SecureRandom; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -183,9 +184,9 @@ public void reset() throws IOException { public List getEncryptionPacketMetadata() { if (false == closed) { - throw new IllegalStateException(); + throw new IllegalStateException("Stream must be closed in order to completely assemble the metadata"); } - return List.copyOf(packetInfoList); + return Collections.unmodifiableList(packetInfoList); } void ensureOpen() throws IOException { From 7e5bb3bb35a7d1897b282ced45bbfe419d965eec Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Fri, 22 Nov 2019 00:18:41 +0200 Subject: [PATCH 27/29] Deleted GCMPacketsCipherInputStream --- .../GCMPacketsCipherInputStream.java | 455 ------------------ .../GCMPacketsCipherInputStreamTests.java | 389 --------------- 2 files changed, 844 deletions(-) delete mode 100644 x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java delete mode 100644 x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java deleted file mode 100644 index 0969c903c6f30..0000000000000 --- a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java +++ /dev/null @@ -1,455 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.repositories.encrypted; - -import org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider; - -import javax.crypto.BadPaddingException; -import javax.crypto.Cipher; -import javax.crypto.IllegalBlockSizeException; -import javax.crypto.SecretKey; -import javax.crypto.ShortBufferException; -import javax.crypto.spec.GCMParameterSpec; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.FilterInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.security.GeneralSecurityException; -import java.security.Provider; -import java.util.Objects; - -import static javax.crypto.Cipher.DECRYPT_MODE; -import static javax.crypto.Cipher.ENCRYPT_MODE; - -/** - * Given an {@code InputStream} and a {@code SecretKey}, creates a wrapper {@code InputStream} that encrypts/decrypts the content of the - * original stream. The original stream must not be otherwise used; it will be closed when the encryptor/decryptor stream is closed. - * - * The method of encryption is AES/GCM/NOPadding. The GCM mode is a form of authenticated encryption, meaning it offers authenticity in - * addition to the expected confidentiality. In other words, during decryption it verifies that the ciphertext being decrypted has not - * been tampered with. - * - * During encryption the source input stream is processed piece-wise and a packet (consisting of multiple pieces) of at most {@code - * GCMPacketsCipherInputStream#PACKET_SIZE_IN_BYTES} bytes size is encrypted and authenticated independently of the other packets. All - * packets in the same stream are encrypted with the same secret key, but a different IV, monotonically increasing with the packet index. - * Consequently, each packet has its own authentication tag appended, even the empty packet (all packets are the same size, but the last - * one can be of any size). The resulting ciphertext has a larger size than the source plaintext. - * - * Decryption also validates the authentication tag. It is important that the {@code Cipher} used during decryption, which is returned by - * the {@code Provider} parameter, NOT internally cache pieces of ciphertext, without releasing the decrypted plaintext, until it - * validates the associated authentication tag. Failure to comply to this requirement, will choke (throw {@code IllegalStateException}) the - * decryption stream, because the ciphertext is processed piece wise, the complete packet is not available fully at one moment. - * - * The resulting decrypted stream will return possibly un-authenticated content, but it is guaranteed that an {@code IOException} is - * thrown, at the latest when the stream has been exhausted ({@code InputStream#read} return {@code -1}) if the ciphertext has been altered. - * - * Both the encrypting and decrypting streams support {@code InputStream#mark} and {@code InputStream#reset}. Because GCM processing cannot - * be reset to a previous state, it only "goes forward", a mark call during the processing of a packet will buffer the processed bytes - * until the next packet boundary. A reset call will pick up any buffered data from a partially processed packet, and re-encrypt the - * following packets (the following packets of a mark call are not buffered, they are re-encrypted). - * - * This is obviously NOT thread-safe. - */ -public class GCMPacketsCipherInputStream extends FilterInputStream { - - static class GCMPacketsWithMarkCipherInputStream extends GCMPacketsCipherInputStream { - - private GCMPacketsWithMarkCipherInputStream(InputStream in, SecretKey secretKey, int mode, long packetIndex, int nonce, - Provider provider) { - super(in, secretKey, mode, packetIndex, nonce, provider); - } - - private ByteArrayOutputStream markWriteBuffer = null; - private ByteArrayInputStream markReadBuffer = new ByteArrayInputStream(new byte[0]); - private boolean markTriggeredForCurrentPacket = false; - private long markPacketIndex; - private int markReadLimit; - - @Override - public int read() throws IOException { - ensureOpen(); - // in case this is a reseted stream that has buffered part of the ciphertext - if (markReadBuffer.available() > 0) { - return markReadBuffer.read(); - } - int cipherByte = super.read(); - // if buffering of the ciphertext is required - if (markTriggeredForCurrentPacket && cipherByte != -1) { - markWriteBuffer.write(cipherByte); - } - return cipherByte; - } - - @Override - public int read(byte b[], int off, int len) throws IOException { - ensureOpen(); - // in case this is a reseted stream that has buffered part of the ciphertext - int bytesReadFromMarkBuffer = markReadBuffer.read(b, off, len); - if (bytesReadFromMarkBuffer != -1) { - return bytesReadFromMarkBuffer; - } - int cipherBytesCount = super.read(b, off, len); - // if buffering of the ciphertext is required - if (markTriggeredForCurrentPacket && cipherBytesCount > 0) { - markWriteBuffer.write(b, off, cipherBytesCount ); - } - return cipherBytesCount; - } - - @Override - public long skip(long n) throws IOException { - ensureOpen(); - if (markReadBuffer.available() > 0) { - return markReadBuffer.skip(n); - } - int bytesAvailable = super.available(); - bytesAvailable = Math.min(bytesAvailable, Math.toIntExact(n)); - if (markTriggeredForCurrentPacket) { - byte[] temp = new byte[bytesAvailable]; - int bytesRead = super.read(temp); - markWriteBuffer.write(temp); - return bytesRead; - } else { - return super.skip(n); - } - } - - @Override - public int available() throws IOException { - ensureOpen(); - return markReadBuffer.available() + super.available(); - } - - @Override - public boolean markSupported() { - return in.markSupported(); - } - - @Override - public void mark(int readLimit) { - markTriggeredForCurrentPacket = true; - markWriteBuffer = new ByteArrayOutputStream(); - if (markReadBuffer.available() > 0) { - markReadBuffer.mark(Integer.MAX_VALUE); - try { - markReadBuffer.transferTo(markWriteBuffer); - } catch (IOException e) { - throw new IllegalStateException(e); - } - markReadBuffer.reset(); - } - markPacketIndex = getPacketIndex(); - markReadLimit = readLimit; - } - - @Override - public void reset() throws IOException { - if (markWriteBuffer == null) { - throw new IOException("mark not called"); - } - if (markPacketIndex > getPacketIndex()) { - throw new IllegalStateException(); - } - // mark triggered before the packet boundary has been read over - if (false == markTriggeredForCurrentPacket) { - // reset underlying input stream to packet boundary - in.reset(); - // set packet index for the next packet and clear any transitory state of any inside of packet processing - setPacketIndex(markPacketIndex); - } - if (markPacketIndex != getPacketIndex()) { - throw new IllegalStateException(); - } - // make any cached ciphertext available to read - markReadBuffer = new ByteArrayInputStream(markWriteBuffer.toByteArray()); - } - - @Override - void readAtTheStartOfPacketHandler() { - if (markTriggeredForCurrentPacket) { - markTriggeredForCurrentPacket = false; - // mark the underlying stream at the start of the packet - in.mark(markReadLimit); - } - } - } - - private static final int GCM_TAG_SIZE_IN_BYTES = 16; - private static final int GCM_IV_SIZE_IN_BYTES = 12; - private static final String GCM_ENCRYPTION_SCHEME = "AES/GCM/NoPadding"; - - static final int PACKET_SIZE_IN_BYTES = 4096; - static final int ENCRYPTED_PACKET_SIZE_IN_BYTES = PACKET_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES; - static final int READ_BUFFER_SIZE_IN_BYTES = 512; - - private boolean done = false; - private boolean closed = false; - private final SecretKey secretKey; - private final int mode; - private final Provider provider; - - private Cipher packetCipher; - private long packetIndex; - private final ByteBuffer packetIV = ByteBuffer.allocate(GCM_IV_SIZE_IN_BYTES); - // how much to read from the underlying stream before finishing the current packet and starting the next one - private int stillToReadInPacket; - private final int packetSizeInBytes; - - private final byte[] inputByteBuffer = new byte[READ_BUFFER_SIZE_IN_BYTES]; - private final byte[] processedByteBuffer = new byte[READ_BUFFER_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES]; - private InputStream processedInputStream = InputStream.nullInputStream(); - private int bytesBufferedInsideTheCipher = 0; - - static GCMPacketsCipherInputStream getEncryptor(InputStream in, SecretKey secretKey, int nonce, Provider provider) { - return new GCMPacketsWithMarkCipherInputStream(in, secretKey, ENCRYPT_MODE, 0, nonce, provider); - } - - static GCMPacketsCipherInputStream getDecryptor(InputStream in, SecretKey secretKey, int nonce, Provider provider) { - return new GCMPacketsWithMarkCipherInputStream(in, secretKey, DECRYPT_MODE, 0, nonce, provider); - } - - public static GCMPacketsCipherInputStream getEncryptor(InputStream in, SecretKey secretKey, int nonce) { - return getEncryptor(in, secretKey, nonce, new BouncyCastleFipsProvider()); - } - - public static GCMPacketsCipherInputStream getDecryptor(InputStream in, SecretKey secretKey, int nonce) { - return getDecryptor(in, secretKey, nonce, new BouncyCastleFipsProvider()); - } - - public static long getEncryptionSizeFromPlainSize(long size) { - if (size < 0) { - throw new IllegalArgumentException(); - } - return (size / PACKET_SIZE_IN_BYTES) * (ENCRYPTED_PACKET_SIZE_IN_BYTES) + (size % PACKET_SIZE_IN_BYTES) + GCM_TAG_SIZE_IN_BYTES; - } - - public static long getDecryptionSizeFromCipherSize(long size) { - if (size < GCM_TAG_SIZE_IN_BYTES) { - throw new IllegalArgumentException(); - } - long plainSize = (size / (ENCRYPTED_PACKET_SIZE_IN_BYTES)) * PACKET_SIZE_IN_BYTES; - if (size % ENCRYPTED_PACKET_SIZE_IN_BYTES < GCM_TAG_SIZE_IN_BYTES) { - throw new IllegalArgumentException(); - } - return plainSize + (size % ENCRYPTED_PACKET_SIZE_IN_BYTES) - GCM_TAG_SIZE_IN_BYTES; - } - - private GCMPacketsCipherInputStream(InputStream in, SecretKey secretKey, int mode, long packetIndex, int nonce, Provider provider) { - super(Objects.requireNonNull(in)); - this.secretKey = Objects.requireNonNull(secretKey); - this.mode = mode; - this.packetIndex = packetIndex; - this.provider = provider; - // the first 8 bytes of the IV for packet encryption are the index of the packet - packetIV.putLong(packetIndex); - // the last 4 bytes of the IV for packet encryption are all equal (randomly generated) - packetIV.putInt(nonce); - if (mode == ENCRYPT_MODE) { - packetSizeInBytes = PACKET_SIZE_IN_BYTES; - } else { - packetSizeInBytes = PACKET_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES; - } - stillToReadInPacket = packetSizeInBytes; - } - - private void reinitPacketCipher() throws GeneralSecurityException { - Cipher cipher; - if (provider != null) { - cipher = Cipher.getInstance(GCM_ENCRYPTION_SCHEME, provider); - } else { - cipher = Cipher.getInstance(GCM_ENCRYPTION_SCHEME); - } - // construct IV and increment packet index - packetIV.putLong(0, Math.incrementExact(packetIndex)); - GCMParameterSpec gcmParameterSpec = new GCMParameterSpec(GCM_TAG_SIZE_IN_BYTES * Byte.SIZE, packetIV.array()); - cipher.init(mode, secretKey, gcmParameterSpec); - packetCipher = cipher; - // the new cipher has no bytes buffered inside - bytesBufferedInsideTheCipher = 0; - } - - private int readAndProcess() throws IOException, GeneralSecurityException { - // do not read anything more, there are still processed bytes to be consumed - if (processedInputStream.available() > 0) { - processedInputStream.available(); - } - // the underlying input stream is exhausted - if (done) { - return -1; - } - // starting to read a new packet - if (stillToReadInPacket == packetSizeInBytes) { - // reinit cipher for this following packet - reinitPacketCipher(); - // call handler to notify subclasses that the processing of a new packet has started - readAtTheStartOfPacketHandler(); - } - int bytesToRead = Math.min(inputByteBuffer.length - bytesBufferedInsideTheCipher, stillToReadInPacket); - if (bytesToRead <= 0) { - throw new IllegalStateException(); - } - int bytesRead = in.read(inputByteBuffer, 0, bytesToRead); - assert bytesRead != 0 : "read must return at least one byte"; - assert processedInputStream.available() == 0 : "there exists processed still to be consumed, but it shouldn't"; - final int bytesProcessed; - if (bytesRead == -1) { - // end of the underlying stream to be encrypted - done = true; - try { - bytesProcessed = packetCipher.doFinal(processedByteBuffer, 0); - } catch (ShortBufferException e) { - throw new IllegalStateException(); - } - // there should be no internally buffered (by the cipher) data remaining after doFinal - bytesBufferedInsideTheCipher -= bytesProcessed; - if (mode == ENCRYPT_MODE) { - bytesBufferedInsideTheCipher += GCM_TAG_SIZE_IN_BYTES; - } else { - bytesBufferedInsideTheCipher -= GCM_TAG_SIZE_IN_BYTES; - } - if (bytesBufferedInsideTheCipher != 0) { - throw new IllegalStateException(); - } - } else { - stillToReadInPacket -= bytesRead; - if (stillToReadInPacket < 0) { - throw new IllegalStateException(); - } - if (stillToReadInPacket == 0) { - // this is the last encryption for this packet - try { - bytesProcessed = packetCipher.doFinal(inputByteBuffer, 0, bytesRead, processedByteBuffer, 0); - } catch (ShortBufferException e) { - throw new IllegalStateException(e); - } - // there should be no internally buffered (by the cipher) data remaining after doFinal - bytesBufferedInsideTheCipher += (bytesRead - bytesProcessed); - if (mode == ENCRYPT_MODE) { - bytesBufferedInsideTheCipher += GCM_TAG_SIZE_IN_BYTES; - } else { - bytesBufferedInsideTheCipher -= GCM_TAG_SIZE_IN_BYTES; - } - if (bytesBufferedInsideTheCipher != 0) { - throw new IllegalArgumentException(); - } - // reset the packet size for the next packet - stillToReadInPacket = packetSizeInBytes; - } else { - // this is a partial encryption inside the packet - try { - bytesProcessed = packetCipher.update(inputByteBuffer, 0, bytesRead, processedByteBuffer, 0); - } catch (ShortBufferException e) { - throw new IllegalStateException(e); - } - // the cipher might encrypt only part of the plaintext and cache the rest - bytesBufferedInsideTheCipher += (bytesRead - bytesProcessed); - } - } - // the "if" here is just an "optimization" - if (bytesProcessed != 0) { - processedInputStream = new ByteArrayInputStream(processedByteBuffer, 0, bytesProcessed); - } - return bytesProcessed; - } - - @Override - public int read() throws IOException { - while (processedInputStream.available() <= 0) { - try { - if (readAndProcess() == -1) { - return -1; - } - } catch (GeneralSecurityException e) { - throw new IOException(e); - } - } - return processedInputStream.read(); - } - - @Override - public int read(byte b[], int off, int len) throws IOException { - while (processedInputStream.available() <= 0) { - try { - if (readAndProcess() == -1) { - return -1; - } - } catch (GeneralSecurityException e) { - throw new IOException(e); - } - } - return processedInputStream.read(b, off, len); - } - - @Override - public long skip(long n) throws IOException { - return processedInputStream.skip(n); - } - - @Override - public int available() throws IOException { - return processedInputStream.available(); - } - - @Override - public void close() throws IOException { - if (closed) { - return; - } - closed = true; - processedInputStream = InputStream.nullInputStream(); - in.close(); - // Throw away the unprocessed data and throw no crypto exceptions. - // Normally the GCM cipher is fully readed before closing, so any authentication - // exceptions would occur while reading. - if (false == done) { - done = true; - try { - packetCipher.doFinal(); - } catch (BadPaddingException | IllegalBlockSizeException ex) { - // Catch exceptions as the rest of the stream is unused. - } - } - } - - @Override - public boolean markSupported() { - return false; - } - - @Override - public void mark(int readLimit) { - } - - @Override - public void reset() throws IOException { - throw new IOException("mark/reset not supported"); - } - - /** - * Sets the packet index and clears the transitory state from processing of the previous packet - */ - void setPacketIndex(long packetIndex) { - processedInputStream = InputStream.nullInputStream(); - stillToReadInPacket = packetSizeInBytes; - done = false; - this.packetIndex = packetIndex; - } - - long getPacketIndex() { - return packetIndex; - } - - void readAtTheStartOfPacketHandler() { - } - - void ensureOpen() throws IOException { - if (closed) { - throw new IOException("Stream closed"); - } - } -} diff --git a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java deleted file mode 100644 index c90f5f3d821ce..0000000000000 --- a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java +++ /dev/null @@ -1,389 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.repositories.encrypted; - -import org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider; -import org.elasticsearch.common.Randomness; -import org.elasticsearch.test.ESTestCase; -import org.hamcrest.Matchers; -import org.junit.Before; -import org.junit.BeforeClass; - -import javax.crypto.AEADBadTagException; -import javax.crypto.KeyGenerator; -import javax.crypto.SecretKey; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.security.AccessController; -import java.security.PrivilegedAction; -import java.security.SecureRandom; -import java.util.Arrays; -import java.util.Random; - -import static org.elasticsearch.repositories.encrypted.GCMPacketsCipherInputStream.ENCRYPTED_PACKET_SIZE_IN_BYTES; -import static org.elasticsearch.repositories.encrypted.GCMPacketsCipherInputStream.READ_BUFFER_SIZE_IN_BYTES; - -public class GCMPacketsCipherInputStreamTests extends ESTestCase { - - private static int TEST_ARRAY_SIZE = 8 * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES; - private static byte[] testPlaintextArray; - private static BouncyCastleFipsProvider bcFipsProvider; - private SecretKey secretKey; - - @BeforeClass - static void setupProvider() { - AccessController.doPrivileged((PrivilegedAction) () -> { - GCMPacketsCipherInputStreamTests.bcFipsProvider = new BouncyCastleFipsProvider(); - return null; - }); - testPlaintextArray = new byte[TEST_ARRAY_SIZE]; - Randomness.get().nextBytes(testPlaintextArray); - } - - @Before - void createSecretKey() throws Exception { - secretKey = generateSecretKey(); - } - - public void testEncryptDecryptEmpty() throws Exception { - testEncryptDecryptRandomOfLength(0, secretKey); - } - - public void testEncryptDecryptSmallerThanBufferSize() throws Exception { - for (int i = 1; i < GCMPacketsCipherInputStream.READ_BUFFER_SIZE_IN_BYTES; i++) { - testEncryptDecryptRandomOfLength(i, secretKey); - } - } - - public void testEncryptDecryptMultipleOfBufferSize() throws Exception { - for (int i = 1; i < 10; i++) { - testEncryptDecryptRandomOfLength(i * GCMPacketsCipherInputStream.READ_BUFFER_SIZE_IN_BYTES, secretKey); - } - } - - public void testEncryptDecryptSmallerThanPacketSize() throws Exception { - for (int i = GCMPacketsCipherInputStream.READ_BUFFER_SIZE_IN_BYTES + 1; i < GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES; i++) { - testEncryptDecryptRandomOfLength(i, secretKey); - } - } - - public void testEncryptDecryptLargerThanPacketSize() throws Exception { - for (int i = GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES + 1; i <= GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES * 3; i++) { - testEncryptDecryptRandomOfLength(i, secretKey); - } - } - - public void testEncryptDecryptMultipleOfPacketSize() throws Exception { - for (int i = 1; i <= 6; i++) { - testEncryptDecryptRandomOfLength(i * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES, secretKey); - } - } - - public void testMarkAndResetAtBeginningForEncryption() throws Exception { - testMarkAndResetToSameOffsetForEncryption(0); - testMarkAndResetToSameOffsetForEncryption(Math.toIntExact(GCMPacketsCipherInputStream. - getEncryptionSizeFromPlainSize(GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES))); - } - - public void testMarkAndResetFirstPacketForEncryption() throws Exception { - for (int i = 1; i < ENCRYPTED_PACKET_SIZE_IN_BYTES; i++) { - testMarkAndResetToSameOffsetForEncryption(i); - } - } - - public void testMarkAndResetRandomSecondPacketForEncryption() throws Exception { - for (int i = ENCRYPTED_PACKET_SIZE_IN_BYTES + 1; i < 2 * ENCRYPTED_PACKET_SIZE_IN_BYTES; i++) { - testMarkAndResetToSameOffsetForEncryption(i); - } - } - - public void testMarkAndResetCrawlForEncryption() throws Exception { - int length = GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES; - int startIndex = randomIntBetween(0, testPlaintextArray.length - length); - int nonce = Randomness.get().nextInt(); - byte[] ciphertextBytes; - try (InputStream cipherInputStream = - GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), - secretKey, nonce, bcFipsProvider)) { - ciphertextBytes = cipherInputStream.readAllBytes(); - } - try (InputStream cipherInputStream = - GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), - secretKey, nonce, bcFipsProvider)) { - cipherInputStream.mark(Integer.MAX_VALUE); - for (int i = 0; i < GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length); i++) { - int skipSize = randomIntBetween(1, Math.toIntExact(GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length)) - i); - // skip bytes - cipherInputStream.readNBytes(skipSize); - cipherInputStream.reset(); - // re-read one byte of the skipped bytes - int byteRead = cipherInputStream.read(); - // mark the one byte progress - cipherInputStream.mark(Integer.MAX_VALUE); - assertThat("Mismatch at position: " + i, (byte) byteRead, Matchers.is(ciphertextBytes[i])); - } - } - } - - public void testMarkAndResetStepInRewindBuffer() throws Exception { - int length = 2 * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES; - int startIndex = randomIntBetween(0, testPlaintextArray.length - length); - int nonce = Randomness.get().nextInt(); - byte[] ciphertextBytes; - try (InputStream cipherInputStream = - GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), - secretKey, nonce, bcFipsProvider)) { - ciphertextBytes = cipherInputStream.readAllBytes(); - } - try (InputStream cipherInputStream = - GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), - secretKey, nonce, bcFipsProvider)) { - int position1 = randomIntBetween(1, ENCRYPTED_PACKET_SIZE_IN_BYTES - 2); - int position2 = randomIntBetween(position1 + 1, ENCRYPTED_PACKET_SIZE_IN_BYTES - 1); - int position3 = ENCRYPTED_PACKET_SIZE_IN_BYTES; - int position4 = randomIntBetween(position3 + 1, 2 * ENCRYPTED_PACKET_SIZE_IN_BYTES - 2); - int position5 = randomIntBetween(position4 + 1, 2 * ENCRYPTED_PACKET_SIZE_IN_BYTES - 1); - int position6 = 2 * ENCRYPTED_PACKET_SIZE_IN_BYTES; - int position7 = Math.toIntExact(GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length)); - // skip position1 bytes - cipherInputStream.readNBytes(position1); - // mark position1 - cipherInputStream.mark(Integer.MAX_VALUE); - byte[] bytesPos17; - if (randomBoolean()) { - bytesPos17 = cipherInputStream.readAllBytes(); - } else { - bytesPos17 = cipherInputStream.readNBytes(position7 - position1); - } - // reset back to position 1 - cipherInputStream.reset(); - byte[] bytesPos12 = cipherInputStream.readNBytes(position2 - position1); - assertTrue(Arrays.equals(bytesPos12, 0, bytesPos12.length, bytesPos17, 0, bytesPos12.length)); - // mark position2 - cipherInputStream.mark(Integer.MAX_VALUE); - byte[] bytesPos26 = cipherInputStream.readNBytes(position6 - position2); - assertTrue(Arrays.equals(bytesPos26, 0, bytesPos26.length, bytesPos17, (position2 - position1), - (position2 - position1) + bytesPos26.length)); - // reset to position 2 - cipherInputStream.reset(); - byte[] bytesPos23 = cipherInputStream.readNBytes(position3 - position2); - assertTrue(Arrays.equals(bytesPos23, 0, bytesPos23.length, bytesPos17, (position2 - position1), - (position2 - position1) + bytesPos23.length)); - // mark position3 - cipherInputStream.mark(Integer.MAX_VALUE); - byte[] bytesPos36 = cipherInputStream.readNBytes(position6 - position3); - assertTrue(Arrays.equals(bytesPos36, 0, bytesPos36.length, bytesPos17, (position3 - position1), - (position3 - position1) + bytesPos36.length)); - // reset to position 3 - cipherInputStream.reset(); - byte[] bytesPos34 = cipherInputStream.readNBytes(position4 - position3); - assertTrue(Arrays.equals(bytesPos34, 0, bytesPos34.length, bytesPos17, (position3 - position1), - (position3 - position1) + bytesPos34.length)); - // mark position4 - cipherInputStream.mark(Integer.MAX_VALUE); - byte[] bytesPos46 = cipherInputStream.readNBytes(position6 - position4); - assertTrue(Arrays.equals(bytesPos46, 0, bytesPos46.length, bytesPos17, (position4 - position1), - (position4 - position1) + bytesPos46.length)); - // reset to position 4 - cipherInputStream.reset(); - byte[] bytesPos45 = cipherInputStream.readNBytes(position5 - position4); - assertTrue(Arrays.equals(bytesPos45, 0, bytesPos45.length, bytesPos17, (position4 - position1), - (position4 - position1) + bytesPos45.length)); - // mark position 5 - cipherInputStream.mark(Integer.MAX_VALUE); - byte[] bytesPos56 = cipherInputStream.readNBytes(position6 - position5); - assertTrue(Arrays.equals(bytesPos56, 0, bytesPos56.length, bytesPos17, (position5 - position1), - (position5 - position1) + bytesPos56.length)); - // mark position 6 - cipherInputStream.mark(Integer.MAX_VALUE); - byte[] bytesPos67; - if (randomBoolean()) { - bytesPos67 = cipherInputStream.readAllBytes(); - } else { - bytesPos67 = cipherInputStream.readNBytes(position7 - position6); - } - assertTrue(Arrays.equals(bytesPos67, 0, bytesPos67.length, bytesPos17, (position6 - position1), - (position6 - position1) + bytesPos67.length)); - // mark position 7 (end of stream) - cipherInputStream.mark(Integer.MAX_VALUE); - // end of stream - assertThat(cipherInputStream.read(), Matchers.is(-1)); - // reset at the end - cipherInputStream.reset(); - assertThat(cipherInputStream.read(), Matchers.is(-1)); - } - } - - public void testDecryptionFails() throws Exception { - Random random = Randomness.get(); - int length = randomIntBetween(0, READ_BUFFER_SIZE_IN_BYTES); - int startIndex = randomIntBetween(0, testPlaintextArray.length - length); - int nonce = random.nextInt(); - byte[] ciphertextBytes; - try (InputStream cipherInputStream = - GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), - secretKey, nonce, bcFipsProvider)) { - ciphertextBytes = cipherInputStream.readAllBytes(); - } - // decryption fails for one byte modifications - for (int i = 0; i < ciphertextBytes.length; i++) { - byte bytei = ciphertextBytes[i]; - while (bytei == ciphertextBytes[i]) { - ciphertextBytes[i] = (byte) random.nextInt(); - } - try (InputStream plainInputStream = - GCMPacketsCipherInputStream.getDecryptor(new ByteArrayInputStream(ciphertextBytes), - secretKey, nonce, bcFipsProvider)) { - IOException e = expectThrows(IOException.class, () -> { - readAllInputStream(plainInputStream, - GCMPacketsCipherInputStream.getDecryptionSizeFromCipherSize(ciphertextBytes.length)); - }); - assertThat(e.getCause(), Matchers.isA(AEADBadTagException.class)); - } - ciphertextBytes[i] = bytei; - } - // decryption fails for one byte omissions - byte[] missingByteCiphertext = new byte[ciphertextBytes.length - 1]; - for (int i = 0; i < ciphertextBytes.length; i++) { - System.arraycopy(ciphertextBytes, 0, missingByteCiphertext, 0, i); - System.arraycopy(ciphertextBytes, i + 1, missingByteCiphertext, i, (ciphertextBytes.length - i - 1)); - try (InputStream plainInputStream = - GCMPacketsCipherInputStream.getDecryptor(new ByteArrayInputStream(missingByteCiphertext), - secretKey, nonce, bcFipsProvider)) { - IOException e = expectThrows(IOException.class, () -> { - readAllInputStream(plainInputStream, - GCMPacketsCipherInputStream.getDecryptionSizeFromCipherSize(missingByteCiphertext.length)); - }); - assertThat(e.getCause(), Matchers.isA(AEADBadTagException.class)); - } - } - // decryption fails for one extra byte - byte[] extraByteCiphertext = new byte[ciphertextBytes.length + 1]; - for (int i = 0; i < ciphertextBytes.length; i++) { - System.arraycopy(ciphertextBytes, 0, extraByteCiphertext, 0, i); - extraByteCiphertext[i] = (byte) random.nextInt(); - System.arraycopy(ciphertextBytes, i, extraByteCiphertext, i + 1, (ciphertextBytes.length - i)); - try (InputStream plainInputStream = - GCMPacketsCipherInputStream.getDecryptor(new ByteArrayInputStream(extraByteCiphertext), - secretKey, nonce, bcFipsProvider)) { - IOException e = expectThrows(IOException.class, () -> { - readAllInputStream(plainInputStream, - GCMPacketsCipherInputStream.getDecryptionSizeFromCipherSize(extraByteCiphertext.length)); - }); - assertThat(e.getCause(), Matchers.isA(AEADBadTagException.class)); - } - } - } - - private void testMarkAndResetToSameOffsetForEncryption(int offset) throws Exception { - int length = 4 * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES + randomIntBetween(0, - GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES); - int startIndex = randomIntBetween(0, testPlaintextArray.length - length); - try (InputStream cipherInputStream = - GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), - secretKey, Randomness.get().nextInt(), bcFipsProvider)) { - // skip offset bytes - cipherInputStream.readNBytes(offset); - // mark after offset - cipherInputStream.mark(Integer.MAX_VALUE); - // read/skip less than (encrypted) packet size - int skipSize = randomIntBetween(1, ENCRYPTED_PACKET_SIZE_IN_BYTES - 1); - byte[] firstPassEncryption = cipherInputStream.readNBytes(skipSize); - // back to start - cipherInputStream.reset(); - // read/skip more than (encrypted) packet size, but less than the full stream - skipSize = randomIntBetween(ENCRYPTED_PACKET_SIZE_IN_BYTES, - Math.toIntExact(GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize((long)length)) - 1 - offset); - byte[] secondPassEncryption = cipherInputStream.readNBytes(skipSize); - assertTrue(Arrays.equals(firstPassEncryption, 0, firstPassEncryption.length, secondPassEncryption, 0, - firstPassEncryption.length)); - // back to start - cipherInputStream.reset(); - byte[] thirdPassEncryption; - // read/skip to end of ciphertext - if (randomBoolean()) { - thirdPassEncryption = cipherInputStream.readAllBytes(); - } else { - thirdPassEncryption = cipherInputStream.readNBytes( - Math.toIntExact(GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize((long) length)) - offset); - } - assertTrue(Arrays.equals(secondPassEncryption, 0, secondPassEncryption.length, thirdPassEncryption, 0, - secondPassEncryption.length)); - // back to start - cipherInputStream.reset(); - // read/skip more than (encrypted) packet size, but less than the full stream - skipSize = randomIntBetween(ENCRYPTED_PACKET_SIZE_IN_BYTES, - Math.toIntExact(GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize((long)length)) - 1 - offset); - byte[] fourthPassEncryption = cipherInputStream.readNBytes(skipSize); - assertTrue(Arrays.equals(fourthPassEncryption, 0, fourthPassEncryption.length, thirdPassEncryption, 0, - fourthPassEncryption.length)); - // back to start - cipherInputStream.reset(); - // read/skip less than (encrypted) packet size - skipSize = randomIntBetween(1, ENCRYPTED_PACKET_SIZE_IN_BYTES - 1); - byte[] fifthsPassEncryption = cipherInputStream.readNBytes(skipSize); - assertTrue(Arrays.equals(fifthsPassEncryption, 0, fifthsPassEncryption.length, fourthPassEncryption, 0, - fifthsPassEncryption.length)); - } - } - - private void testEncryptDecryptRandomOfLength(int length, SecretKey secretKey) throws Exception { - int nonce = Randomness.get().nextInt(); - ByteArrayOutputStream cipherTextOutput; - ByteArrayOutputStream plainTextOutput; - int startIndex = randomIntBetween(0, testPlaintextArray.length - length); - // encrypt - try (InputStream cipherInputStream = - GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), - secretKey, nonce, bcFipsProvider)) { - cipherTextOutput = readAllInputStream(cipherInputStream, GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length)); - } - //decrypt - try (InputStream plainInputStream = - GCMPacketsCipherInputStream.getDecryptor(new ByteArrayInputStream(cipherTextOutput.toByteArray()), - secretKey, nonce, bcFipsProvider)) { - plainTextOutput = readAllInputStream(plainInputStream, - GCMPacketsCipherInputStream.getDecryptionSizeFromCipherSize(cipherTextOutput.size())); - } - assertTrue(Arrays.equals(plainTextOutput.toByteArray(), 0, length, testPlaintextArray, startIndex, startIndex + length)); - } - - private SecretKey generateSecretKey() throws Exception { - return AccessController.doPrivileged((PrivilegedAction) () -> { - try { - KeyGenerator keyGen = KeyGenerator.getInstance("AES", bcFipsProvider); - keyGen.init(256, SecureRandom.getInstance("DEFAULT", bcFipsProvider)); - return keyGen.generateKey(); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - } - - // read "adversarily" in small random pieces - private ByteArrayOutputStream readAllInputStream(InputStream inputStream, long size) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(Math.toIntExact(size)); - byte[] temp = new byte[randomIntBetween(1, size != 0 ? Math.toIntExact(size) : 1)]; - do { - int bytesRead = inputStream.read(temp, 0, randomIntBetween(1, temp.length)); - if (bytesRead == -1) { - break; - } - baos.write(temp, 0, bytesRead); - if (randomBoolean()) { - int singleByte = inputStream.read(); - if (singleByte == -1) { - break; - } - baos.write(singleByte); - } - } while (true); - assertThat(baos.size(), Matchers.is(Math.toIntExact(size))); - return baos; - } -} From 58ea4f3f634bba053fb956fbf793918d77fe7cf2 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Fri, 22 Nov 2019 15:45:55 +0200 Subject: [PATCH 28/29] Revert "Deleted GCMPacketsCipherInputStream" This reverts commit 7e5bb3bb35a7d1897b282ced45bbfe419d965eec. --- .../GCMPacketsCipherInputStream.java | 455 ++++++++++++++++++ .../GCMPacketsCipherInputStreamTests.java | 389 +++++++++++++++ 2 files changed, 844 insertions(+) create mode 100644 x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java create mode 100644 x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java new file mode 100644 index 0000000000000..0969c903c6f30 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java @@ -0,0 +1,455 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.repositories.encrypted; + +import org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider; + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.SecretKey; +import javax.crypto.ShortBufferException; +import javax.crypto.spec.GCMParameterSpec; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.security.GeneralSecurityException; +import java.security.Provider; +import java.util.Objects; + +import static javax.crypto.Cipher.DECRYPT_MODE; +import static javax.crypto.Cipher.ENCRYPT_MODE; + +/** + * Given an {@code InputStream} and a {@code SecretKey}, creates a wrapper {@code InputStream} that encrypts/decrypts the content of the + * original stream. The original stream must not be otherwise used; it will be closed when the encryptor/decryptor stream is closed. + * + * The method of encryption is AES/GCM/NOPadding. The GCM mode is a form of authenticated encryption, meaning it offers authenticity in + * addition to the expected confidentiality. In other words, during decryption it verifies that the ciphertext being decrypted has not + * been tampered with. + * + * During encryption the source input stream is processed piece-wise and a packet (consisting of multiple pieces) of at most {@code + * GCMPacketsCipherInputStream#PACKET_SIZE_IN_BYTES} bytes size is encrypted and authenticated independently of the other packets. All + * packets in the same stream are encrypted with the same secret key, but a different IV, monotonically increasing with the packet index. + * Consequently, each packet has its own authentication tag appended, even the empty packet (all packets are the same size, but the last + * one can be of any size). The resulting ciphertext has a larger size than the source plaintext. + * + * Decryption also validates the authentication tag. It is important that the {@code Cipher} used during decryption, which is returned by + * the {@code Provider} parameter, NOT internally cache pieces of ciphertext, without releasing the decrypted plaintext, until it + * validates the associated authentication tag. Failure to comply to this requirement, will choke (throw {@code IllegalStateException}) the + * decryption stream, because the ciphertext is processed piece wise, the complete packet is not available fully at one moment. + * + * The resulting decrypted stream will return possibly un-authenticated content, but it is guaranteed that an {@code IOException} is + * thrown, at the latest when the stream has been exhausted ({@code InputStream#read} return {@code -1}) if the ciphertext has been altered. + * + * Both the encrypting and decrypting streams support {@code InputStream#mark} and {@code InputStream#reset}. Because GCM processing cannot + * be reset to a previous state, it only "goes forward", a mark call during the processing of a packet will buffer the processed bytes + * until the next packet boundary. A reset call will pick up any buffered data from a partially processed packet, and re-encrypt the + * following packets (the following packets of a mark call are not buffered, they are re-encrypted). + * + * This is obviously NOT thread-safe. + */ +public class GCMPacketsCipherInputStream extends FilterInputStream { + + static class GCMPacketsWithMarkCipherInputStream extends GCMPacketsCipherInputStream { + + private GCMPacketsWithMarkCipherInputStream(InputStream in, SecretKey secretKey, int mode, long packetIndex, int nonce, + Provider provider) { + super(in, secretKey, mode, packetIndex, nonce, provider); + } + + private ByteArrayOutputStream markWriteBuffer = null; + private ByteArrayInputStream markReadBuffer = new ByteArrayInputStream(new byte[0]); + private boolean markTriggeredForCurrentPacket = false; + private long markPacketIndex; + private int markReadLimit; + + @Override + public int read() throws IOException { + ensureOpen(); + // in case this is a reseted stream that has buffered part of the ciphertext + if (markReadBuffer.available() > 0) { + return markReadBuffer.read(); + } + int cipherByte = super.read(); + // if buffering of the ciphertext is required + if (markTriggeredForCurrentPacket && cipherByte != -1) { + markWriteBuffer.write(cipherByte); + } + return cipherByte; + } + + @Override + public int read(byte b[], int off, int len) throws IOException { + ensureOpen(); + // in case this is a reseted stream that has buffered part of the ciphertext + int bytesReadFromMarkBuffer = markReadBuffer.read(b, off, len); + if (bytesReadFromMarkBuffer != -1) { + return bytesReadFromMarkBuffer; + } + int cipherBytesCount = super.read(b, off, len); + // if buffering of the ciphertext is required + if (markTriggeredForCurrentPacket && cipherBytesCount > 0) { + markWriteBuffer.write(b, off, cipherBytesCount ); + } + return cipherBytesCount; + } + + @Override + public long skip(long n) throws IOException { + ensureOpen(); + if (markReadBuffer.available() > 0) { + return markReadBuffer.skip(n); + } + int bytesAvailable = super.available(); + bytesAvailable = Math.min(bytesAvailable, Math.toIntExact(n)); + if (markTriggeredForCurrentPacket) { + byte[] temp = new byte[bytesAvailable]; + int bytesRead = super.read(temp); + markWriteBuffer.write(temp); + return bytesRead; + } else { + return super.skip(n); + } + } + + @Override + public int available() throws IOException { + ensureOpen(); + return markReadBuffer.available() + super.available(); + } + + @Override + public boolean markSupported() { + return in.markSupported(); + } + + @Override + public void mark(int readLimit) { + markTriggeredForCurrentPacket = true; + markWriteBuffer = new ByteArrayOutputStream(); + if (markReadBuffer.available() > 0) { + markReadBuffer.mark(Integer.MAX_VALUE); + try { + markReadBuffer.transferTo(markWriteBuffer); + } catch (IOException e) { + throw new IllegalStateException(e); + } + markReadBuffer.reset(); + } + markPacketIndex = getPacketIndex(); + markReadLimit = readLimit; + } + + @Override + public void reset() throws IOException { + if (markWriteBuffer == null) { + throw new IOException("mark not called"); + } + if (markPacketIndex > getPacketIndex()) { + throw new IllegalStateException(); + } + // mark triggered before the packet boundary has been read over + if (false == markTriggeredForCurrentPacket) { + // reset underlying input stream to packet boundary + in.reset(); + // set packet index for the next packet and clear any transitory state of any inside of packet processing + setPacketIndex(markPacketIndex); + } + if (markPacketIndex != getPacketIndex()) { + throw new IllegalStateException(); + } + // make any cached ciphertext available to read + markReadBuffer = new ByteArrayInputStream(markWriteBuffer.toByteArray()); + } + + @Override + void readAtTheStartOfPacketHandler() { + if (markTriggeredForCurrentPacket) { + markTriggeredForCurrentPacket = false; + // mark the underlying stream at the start of the packet + in.mark(markReadLimit); + } + } + } + + private static final int GCM_TAG_SIZE_IN_BYTES = 16; + private static final int GCM_IV_SIZE_IN_BYTES = 12; + private static final String GCM_ENCRYPTION_SCHEME = "AES/GCM/NoPadding"; + + static final int PACKET_SIZE_IN_BYTES = 4096; + static final int ENCRYPTED_PACKET_SIZE_IN_BYTES = PACKET_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES; + static final int READ_BUFFER_SIZE_IN_BYTES = 512; + + private boolean done = false; + private boolean closed = false; + private final SecretKey secretKey; + private final int mode; + private final Provider provider; + + private Cipher packetCipher; + private long packetIndex; + private final ByteBuffer packetIV = ByteBuffer.allocate(GCM_IV_SIZE_IN_BYTES); + // how much to read from the underlying stream before finishing the current packet and starting the next one + private int stillToReadInPacket; + private final int packetSizeInBytes; + + private final byte[] inputByteBuffer = new byte[READ_BUFFER_SIZE_IN_BYTES]; + private final byte[] processedByteBuffer = new byte[READ_BUFFER_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES]; + private InputStream processedInputStream = InputStream.nullInputStream(); + private int bytesBufferedInsideTheCipher = 0; + + static GCMPacketsCipherInputStream getEncryptor(InputStream in, SecretKey secretKey, int nonce, Provider provider) { + return new GCMPacketsWithMarkCipherInputStream(in, secretKey, ENCRYPT_MODE, 0, nonce, provider); + } + + static GCMPacketsCipherInputStream getDecryptor(InputStream in, SecretKey secretKey, int nonce, Provider provider) { + return new GCMPacketsWithMarkCipherInputStream(in, secretKey, DECRYPT_MODE, 0, nonce, provider); + } + + public static GCMPacketsCipherInputStream getEncryptor(InputStream in, SecretKey secretKey, int nonce) { + return getEncryptor(in, secretKey, nonce, new BouncyCastleFipsProvider()); + } + + public static GCMPacketsCipherInputStream getDecryptor(InputStream in, SecretKey secretKey, int nonce) { + return getDecryptor(in, secretKey, nonce, new BouncyCastleFipsProvider()); + } + + public static long getEncryptionSizeFromPlainSize(long size) { + if (size < 0) { + throw new IllegalArgumentException(); + } + return (size / PACKET_SIZE_IN_BYTES) * (ENCRYPTED_PACKET_SIZE_IN_BYTES) + (size % PACKET_SIZE_IN_BYTES) + GCM_TAG_SIZE_IN_BYTES; + } + + public static long getDecryptionSizeFromCipherSize(long size) { + if (size < GCM_TAG_SIZE_IN_BYTES) { + throw new IllegalArgumentException(); + } + long plainSize = (size / (ENCRYPTED_PACKET_SIZE_IN_BYTES)) * PACKET_SIZE_IN_BYTES; + if (size % ENCRYPTED_PACKET_SIZE_IN_BYTES < GCM_TAG_SIZE_IN_BYTES) { + throw new IllegalArgumentException(); + } + return plainSize + (size % ENCRYPTED_PACKET_SIZE_IN_BYTES) - GCM_TAG_SIZE_IN_BYTES; + } + + private GCMPacketsCipherInputStream(InputStream in, SecretKey secretKey, int mode, long packetIndex, int nonce, Provider provider) { + super(Objects.requireNonNull(in)); + this.secretKey = Objects.requireNonNull(secretKey); + this.mode = mode; + this.packetIndex = packetIndex; + this.provider = provider; + // the first 8 bytes of the IV for packet encryption are the index of the packet + packetIV.putLong(packetIndex); + // the last 4 bytes of the IV for packet encryption are all equal (randomly generated) + packetIV.putInt(nonce); + if (mode == ENCRYPT_MODE) { + packetSizeInBytes = PACKET_SIZE_IN_BYTES; + } else { + packetSizeInBytes = PACKET_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES; + } + stillToReadInPacket = packetSizeInBytes; + } + + private void reinitPacketCipher() throws GeneralSecurityException { + Cipher cipher; + if (provider != null) { + cipher = Cipher.getInstance(GCM_ENCRYPTION_SCHEME, provider); + } else { + cipher = Cipher.getInstance(GCM_ENCRYPTION_SCHEME); + } + // construct IV and increment packet index + packetIV.putLong(0, Math.incrementExact(packetIndex)); + GCMParameterSpec gcmParameterSpec = new GCMParameterSpec(GCM_TAG_SIZE_IN_BYTES * Byte.SIZE, packetIV.array()); + cipher.init(mode, secretKey, gcmParameterSpec); + packetCipher = cipher; + // the new cipher has no bytes buffered inside + bytesBufferedInsideTheCipher = 0; + } + + private int readAndProcess() throws IOException, GeneralSecurityException { + // do not read anything more, there are still processed bytes to be consumed + if (processedInputStream.available() > 0) { + processedInputStream.available(); + } + // the underlying input stream is exhausted + if (done) { + return -1; + } + // starting to read a new packet + if (stillToReadInPacket == packetSizeInBytes) { + // reinit cipher for this following packet + reinitPacketCipher(); + // call handler to notify subclasses that the processing of a new packet has started + readAtTheStartOfPacketHandler(); + } + int bytesToRead = Math.min(inputByteBuffer.length - bytesBufferedInsideTheCipher, stillToReadInPacket); + if (bytesToRead <= 0) { + throw new IllegalStateException(); + } + int bytesRead = in.read(inputByteBuffer, 0, bytesToRead); + assert bytesRead != 0 : "read must return at least one byte"; + assert processedInputStream.available() == 0 : "there exists processed still to be consumed, but it shouldn't"; + final int bytesProcessed; + if (bytesRead == -1) { + // end of the underlying stream to be encrypted + done = true; + try { + bytesProcessed = packetCipher.doFinal(processedByteBuffer, 0); + } catch (ShortBufferException e) { + throw new IllegalStateException(); + } + // there should be no internally buffered (by the cipher) data remaining after doFinal + bytesBufferedInsideTheCipher -= bytesProcessed; + if (mode == ENCRYPT_MODE) { + bytesBufferedInsideTheCipher += GCM_TAG_SIZE_IN_BYTES; + } else { + bytesBufferedInsideTheCipher -= GCM_TAG_SIZE_IN_BYTES; + } + if (bytesBufferedInsideTheCipher != 0) { + throw new IllegalStateException(); + } + } else { + stillToReadInPacket -= bytesRead; + if (stillToReadInPacket < 0) { + throw new IllegalStateException(); + } + if (stillToReadInPacket == 0) { + // this is the last encryption for this packet + try { + bytesProcessed = packetCipher.doFinal(inputByteBuffer, 0, bytesRead, processedByteBuffer, 0); + } catch (ShortBufferException e) { + throw new IllegalStateException(e); + } + // there should be no internally buffered (by the cipher) data remaining after doFinal + bytesBufferedInsideTheCipher += (bytesRead - bytesProcessed); + if (mode == ENCRYPT_MODE) { + bytesBufferedInsideTheCipher += GCM_TAG_SIZE_IN_BYTES; + } else { + bytesBufferedInsideTheCipher -= GCM_TAG_SIZE_IN_BYTES; + } + if (bytesBufferedInsideTheCipher != 0) { + throw new IllegalArgumentException(); + } + // reset the packet size for the next packet + stillToReadInPacket = packetSizeInBytes; + } else { + // this is a partial encryption inside the packet + try { + bytesProcessed = packetCipher.update(inputByteBuffer, 0, bytesRead, processedByteBuffer, 0); + } catch (ShortBufferException e) { + throw new IllegalStateException(e); + } + // the cipher might encrypt only part of the plaintext and cache the rest + bytesBufferedInsideTheCipher += (bytesRead - bytesProcessed); + } + } + // the "if" here is just an "optimization" + if (bytesProcessed != 0) { + processedInputStream = new ByteArrayInputStream(processedByteBuffer, 0, bytesProcessed); + } + return bytesProcessed; + } + + @Override + public int read() throws IOException { + while (processedInputStream.available() <= 0) { + try { + if (readAndProcess() == -1) { + return -1; + } + } catch (GeneralSecurityException e) { + throw new IOException(e); + } + } + return processedInputStream.read(); + } + + @Override + public int read(byte b[], int off, int len) throws IOException { + while (processedInputStream.available() <= 0) { + try { + if (readAndProcess() == -1) { + return -1; + } + } catch (GeneralSecurityException e) { + throw new IOException(e); + } + } + return processedInputStream.read(b, off, len); + } + + @Override + public long skip(long n) throws IOException { + return processedInputStream.skip(n); + } + + @Override + public int available() throws IOException { + return processedInputStream.available(); + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + closed = true; + processedInputStream = InputStream.nullInputStream(); + in.close(); + // Throw away the unprocessed data and throw no crypto exceptions. + // Normally the GCM cipher is fully readed before closing, so any authentication + // exceptions would occur while reading. + if (false == done) { + done = true; + try { + packetCipher.doFinal(); + } catch (BadPaddingException | IllegalBlockSizeException ex) { + // Catch exceptions as the rest of the stream is unused. + } + } + } + + @Override + public boolean markSupported() { + return false; + } + + @Override + public void mark(int readLimit) { + } + + @Override + public void reset() throws IOException { + throw new IOException("mark/reset not supported"); + } + + /** + * Sets the packet index and clears the transitory state from processing of the previous packet + */ + void setPacketIndex(long packetIndex) { + processedInputStream = InputStream.nullInputStream(); + stillToReadInPacket = packetSizeInBytes; + done = false; + this.packetIndex = packetIndex; + } + + long getPacketIndex() { + return packetIndex; + } + + void readAtTheStartOfPacketHandler() { + } + + void ensureOpen() throws IOException { + if (closed) { + throw new IOException("Stream closed"); + } + } +} diff --git a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java new file mode 100644 index 0000000000000..c90f5f3d821ce --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java @@ -0,0 +1,389 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.repositories.encrypted; + +import org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider; +import org.elasticsearch.common.Randomness; +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; +import org.junit.Before; +import org.junit.BeforeClass; + +import javax.crypto.AEADBadTagException; +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.security.SecureRandom; +import java.util.Arrays; +import java.util.Random; + +import static org.elasticsearch.repositories.encrypted.GCMPacketsCipherInputStream.ENCRYPTED_PACKET_SIZE_IN_BYTES; +import static org.elasticsearch.repositories.encrypted.GCMPacketsCipherInputStream.READ_BUFFER_SIZE_IN_BYTES; + +public class GCMPacketsCipherInputStreamTests extends ESTestCase { + + private static int TEST_ARRAY_SIZE = 8 * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES; + private static byte[] testPlaintextArray; + private static BouncyCastleFipsProvider bcFipsProvider; + private SecretKey secretKey; + + @BeforeClass + static void setupProvider() { + AccessController.doPrivileged((PrivilegedAction) () -> { + GCMPacketsCipherInputStreamTests.bcFipsProvider = new BouncyCastleFipsProvider(); + return null; + }); + testPlaintextArray = new byte[TEST_ARRAY_SIZE]; + Randomness.get().nextBytes(testPlaintextArray); + } + + @Before + void createSecretKey() throws Exception { + secretKey = generateSecretKey(); + } + + public void testEncryptDecryptEmpty() throws Exception { + testEncryptDecryptRandomOfLength(0, secretKey); + } + + public void testEncryptDecryptSmallerThanBufferSize() throws Exception { + for (int i = 1; i < GCMPacketsCipherInputStream.READ_BUFFER_SIZE_IN_BYTES; i++) { + testEncryptDecryptRandomOfLength(i, secretKey); + } + } + + public void testEncryptDecryptMultipleOfBufferSize() throws Exception { + for (int i = 1; i < 10; i++) { + testEncryptDecryptRandomOfLength(i * GCMPacketsCipherInputStream.READ_BUFFER_SIZE_IN_BYTES, secretKey); + } + } + + public void testEncryptDecryptSmallerThanPacketSize() throws Exception { + for (int i = GCMPacketsCipherInputStream.READ_BUFFER_SIZE_IN_BYTES + 1; i < GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES; i++) { + testEncryptDecryptRandomOfLength(i, secretKey); + } + } + + public void testEncryptDecryptLargerThanPacketSize() throws Exception { + for (int i = GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES + 1; i <= GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES * 3; i++) { + testEncryptDecryptRandomOfLength(i, secretKey); + } + } + + public void testEncryptDecryptMultipleOfPacketSize() throws Exception { + for (int i = 1; i <= 6; i++) { + testEncryptDecryptRandomOfLength(i * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES, secretKey); + } + } + + public void testMarkAndResetAtBeginningForEncryption() throws Exception { + testMarkAndResetToSameOffsetForEncryption(0); + testMarkAndResetToSameOffsetForEncryption(Math.toIntExact(GCMPacketsCipherInputStream. + getEncryptionSizeFromPlainSize(GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES))); + } + + public void testMarkAndResetFirstPacketForEncryption() throws Exception { + for (int i = 1; i < ENCRYPTED_PACKET_SIZE_IN_BYTES; i++) { + testMarkAndResetToSameOffsetForEncryption(i); + } + } + + public void testMarkAndResetRandomSecondPacketForEncryption() throws Exception { + for (int i = ENCRYPTED_PACKET_SIZE_IN_BYTES + 1; i < 2 * ENCRYPTED_PACKET_SIZE_IN_BYTES; i++) { + testMarkAndResetToSameOffsetForEncryption(i); + } + } + + public void testMarkAndResetCrawlForEncryption() throws Exception { + int length = GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES; + int startIndex = randomIntBetween(0, testPlaintextArray.length - length); + int nonce = Randomness.get().nextInt(); + byte[] ciphertextBytes; + try (InputStream cipherInputStream = + GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), + secretKey, nonce, bcFipsProvider)) { + ciphertextBytes = cipherInputStream.readAllBytes(); + } + try (InputStream cipherInputStream = + GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), + secretKey, nonce, bcFipsProvider)) { + cipherInputStream.mark(Integer.MAX_VALUE); + for (int i = 0; i < GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length); i++) { + int skipSize = randomIntBetween(1, Math.toIntExact(GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length)) - i); + // skip bytes + cipherInputStream.readNBytes(skipSize); + cipherInputStream.reset(); + // re-read one byte of the skipped bytes + int byteRead = cipherInputStream.read(); + // mark the one byte progress + cipherInputStream.mark(Integer.MAX_VALUE); + assertThat("Mismatch at position: " + i, (byte) byteRead, Matchers.is(ciphertextBytes[i])); + } + } + } + + public void testMarkAndResetStepInRewindBuffer() throws Exception { + int length = 2 * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES; + int startIndex = randomIntBetween(0, testPlaintextArray.length - length); + int nonce = Randomness.get().nextInt(); + byte[] ciphertextBytes; + try (InputStream cipherInputStream = + GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), + secretKey, nonce, bcFipsProvider)) { + ciphertextBytes = cipherInputStream.readAllBytes(); + } + try (InputStream cipherInputStream = + GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), + secretKey, nonce, bcFipsProvider)) { + int position1 = randomIntBetween(1, ENCRYPTED_PACKET_SIZE_IN_BYTES - 2); + int position2 = randomIntBetween(position1 + 1, ENCRYPTED_PACKET_SIZE_IN_BYTES - 1); + int position3 = ENCRYPTED_PACKET_SIZE_IN_BYTES; + int position4 = randomIntBetween(position3 + 1, 2 * ENCRYPTED_PACKET_SIZE_IN_BYTES - 2); + int position5 = randomIntBetween(position4 + 1, 2 * ENCRYPTED_PACKET_SIZE_IN_BYTES - 1); + int position6 = 2 * ENCRYPTED_PACKET_SIZE_IN_BYTES; + int position7 = Math.toIntExact(GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length)); + // skip position1 bytes + cipherInputStream.readNBytes(position1); + // mark position1 + cipherInputStream.mark(Integer.MAX_VALUE); + byte[] bytesPos17; + if (randomBoolean()) { + bytesPos17 = cipherInputStream.readAllBytes(); + } else { + bytesPos17 = cipherInputStream.readNBytes(position7 - position1); + } + // reset back to position 1 + cipherInputStream.reset(); + byte[] bytesPos12 = cipherInputStream.readNBytes(position2 - position1); + assertTrue(Arrays.equals(bytesPos12, 0, bytesPos12.length, bytesPos17, 0, bytesPos12.length)); + // mark position2 + cipherInputStream.mark(Integer.MAX_VALUE); + byte[] bytesPos26 = cipherInputStream.readNBytes(position6 - position2); + assertTrue(Arrays.equals(bytesPos26, 0, bytesPos26.length, bytesPos17, (position2 - position1), + (position2 - position1) + bytesPos26.length)); + // reset to position 2 + cipherInputStream.reset(); + byte[] bytesPos23 = cipherInputStream.readNBytes(position3 - position2); + assertTrue(Arrays.equals(bytesPos23, 0, bytesPos23.length, bytesPos17, (position2 - position1), + (position2 - position1) + bytesPos23.length)); + // mark position3 + cipherInputStream.mark(Integer.MAX_VALUE); + byte[] bytesPos36 = cipherInputStream.readNBytes(position6 - position3); + assertTrue(Arrays.equals(bytesPos36, 0, bytesPos36.length, bytesPos17, (position3 - position1), + (position3 - position1) + bytesPos36.length)); + // reset to position 3 + cipherInputStream.reset(); + byte[] bytesPos34 = cipherInputStream.readNBytes(position4 - position3); + assertTrue(Arrays.equals(bytesPos34, 0, bytesPos34.length, bytesPos17, (position3 - position1), + (position3 - position1) + bytesPos34.length)); + // mark position4 + cipherInputStream.mark(Integer.MAX_VALUE); + byte[] bytesPos46 = cipherInputStream.readNBytes(position6 - position4); + assertTrue(Arrays.equals(bytesPos46, 0, bytesPos46.length, bytesPos17, (position4 - position1), + (position4 - position1) + bytesPos46.length)); + // reset to position 4 + cipherInputStream.reset(); + byte[] bytesPos45 = cipherInputStream.readNBytes(position5 - position4); + assertTrue(Arrays.equals(bytesPos45, 0, bytesPos45.length, bytesPos17, (position4 - position1), + (position4 - position1) + bytesPos45.length)); + // mark position 5 + cipherInputStream.mark(Integer.MAX_VALUE); + byte[] bytesPos56 = cipherInputStream.readNBytes(position6 - position5); + assertTrue(Arrays.equals(bytesPos56, 0, bytesPos56.length, bytesPos17, (position5 - position1), + (position5 - position1) + bytesPos56.length)); + // mark position 6 + cipherInputStream.mark(Integer.MAX_VALUE); + byte[] bytesPos67; + if (randomBoolean()) { + bytesPos67 = cipherInputStream.readAllBytes(); + } else { + bytesPos67 = cipherInputStream.readNBytes(position7 - position6); + } + assertTrue(Arrays.equals(bytesPos67, 0, bytesPos67.length, bytesPos17, (position6 - position1), + (position6 - position1) + bytesPos67.length)); + // mark position 7 (end of stream) + cipherInputStream.mark(Integer.MAX_VALUE); + // end of stream + assertThat(cipherInputStream.read(), Matchers.is(-1)); + // reset at the end + cipherInputStream.reset(); + assertThat(cipherInputStream.read(), Matchers.is(-1)); + } + } + + public void testDecryptionFails() throws Exception { + Random random = Randomness.get(); + int length = randomIntBetween(0, READ_BUFFER_SIZE_IN_BYTES); + int startIndex = randomIntBetween(0, testPlaintextArray.length - length); + int nonce = random.nextInt(); + byte[] ciphertextBytes; + try (InputStream cipherInputStream = + GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), + secretKey, nonce, bcFipsProvider)) { + ciphertextBytes = cipherInputStream.readAllBytes(); + } + // decryption fails for one byte modifications + for (int i = 0; i < ciphertextBytes.length; i++) { + byte bytei = ciphertextBytes[i]; + while (bytei == ciphertextBytes[i]) { + ciphertextBytes[i] = (byte) random.nextInt(); + } + try (InputStream plainInputStream = + GCMPacketsCipherInputStream.getDecryptor(new ByteArrayInputStream(ciphertextBytes), + secretKey, nonce, bcFipsProvider)) { + IOException e = expectThrows(IOException.class, () -> { + readAllInputStream(plainInputStream, + GCMPacketsCipherInputStream.getDecryptionSizeFromCipherSize(ciphertextBytes.length)); + }); + assertThat(e.getCause(), Matchers.isA(AEADBadTagException.class)); + } + ciphertextBytes[i] = bytei; + } + // decryption fails for one byte omissions + byte[] missingByteCiphertext = new byte[ciphertextBytes.length - 1]; + for (int i = 0; i < ciphertextBytes.length; i++) { + System.arraycopy(ciphertextBytes, 0, missingByteCiphertext, 0, i); + System.arraycopy(ciphertextBytes, i + 1, missingByteCiphertext, i, (ciphertextBytes.length - i - 1)); + try (InputStream plainInputStream = + GCMPacketsCipherInputStream.getDecryptor(new ByteArrayInputStream(missingByteCiphertext), + secretKey, nonce, bcFipsProvider)) { + IOException e = expectThrows(IOException.class, () -> { + readAllInputStream(plainInputStream, + GCMPacketsCipherInputStream.getDecryptionSizeFromCipherSize(missingByteCiphertext.length)); + }); + assertThat(e.getCause(), Matchers.isA(AEADBadTagException.class)); + } + } + // decryption fails for one extra byte + byte[] extraByteCiphertext = new byte[ciphertextBytes.length + 1]; + for (int i = 0; i < ciphertextBytes.length; i++) { + System.arraycopy(ciphertextBytes, 0, extraByteCiphertext, 0, i); + extraByteCiphertext[i] = (byte) random.nextInt(); + System.arraycopy(ciphertextBytes, i, extraByteCiphertext, i + 1, (ciphertextBytes.length - i)); + try (InputStream plainInputStream = + GCMPacketsCipherInputStream.getDecryptor(new ByteArrayInputStream(extraByteCiphertext), + secretKey, nonce, bcFipsProvider)) { + IOException e = expectThrows(IOException.class, () -> { + readAllInputStream(plainInputStream, + GCMPacketsCipherInputStream.getDecryptionSizeFromCipherSize(extraByteCiphertext.length)); + }); + assertThat(e.getCause(), Matchers.isA(AEADBadTagException.class)); + } + } + } + + private void testMarkAndResetToSameOffsetForEncryption(int offset) throws Exception { + int length = 4 * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES + randomIntBetween(0, + GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES); + int startIndex = randomIntBetween(0, testPlaintextArray.length - length); + try (InputStream cipherInputStream = + GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), + secretKey, Randomness.get().nextInt(), bcFipsProvider)) { + // skip offset bytes + cipherInputStream.readNBytes(offset); + // mark after offset + cipherInputStream.mark(Integer.MAX_VALUE); + // read/skip less than (encrypted) packet size + int skipSize = randomIntBetween(1, ENCRYPTED_PACKET_SIZE_IN_BYTES - 1); + byte[] firstPassEncryption = cipherInputStream.readNBytes(skipSize); + // back to start + cipherInputStream.reset(); + // read/skip more than (encrypted) packet size, but less than the full stream + skipSize = randomIntBetween(ENCRYPTED_PACKET_SIZE_IN_BYTES, + Math.toIntExact(GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize((long)length)) - 1 - offset); + byte[] secondPassEncryption = cipherInputStream.readNBytes(skipSize); + assertTrue(Arrays.equals(firstPassEncryption, 0, firstPassEncryption.length, secondPassEncryption, 0, + firstPassEncryption.length)); + // back to start + cipherInputStream.reset(); + byte[] thirdPassEncryption; + // read/skip to end of ciphertext + if (randomBoolean()) { + thirdPassEncryption = cipherInputStream.readAllBytes(); + } else { + thirdPassEncryption = cipherInputStream.readNBytes( + Math.toIntExact(GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize((long) length)) - offset); + } + assertTrue(Arrays.equals(secondPassEncryption, 0, secondPassEncryption.length, thirdPassEncryption, 0, + secondPassEncryption.length)); + // back to start + cipherInputStream.reset(); + // read/skip more than (encrypted) packet size, but less than the full stream + skipSize = randomIntBetween(ENCRYPTED_PACKET_SIZE_IN_BYTES, + Math.toIntExact(GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize((long)length)) - 1 - offset); + byte[] fourthPassEncryption = cipherInputStream.readNBytes(skipSize); + assertTrue(Arrays.equals(fourthPassEncryption, 0, fourthPassEncryption.length, thirdPassEncryption, 0, + fourthPassEncryption.length)); + // back to start + cipherInputStream.reset(); + // read/skip less than (encrypted) packet size + skipSize = randomIntBetween(1, ENCRYPTED_PACKET_SIZE_IN_BYTES - 1); + byte[] fifthsPassEncryption = cipherInputStream.readNBytes(skipSize); + assertTrue(Arrays.equals(fifthsPassEncryption, 0, fifthsPassEncryption.length, fourthPassEncryption, 0, + fifthsPassEncryption.length)); + } + } + + private void testEncryptDecryptRandomOfLength(int length, SecretKey secretKey) throws Exception { + int nonce = Randomness.get().nextInt(); + ByteArrayOutputStream cipherTextOutput; + ByteArrayOutputStream plainTextOutput; + int startIndex = randomIntBetween(0, testPlaintextArray.length - length); + // encrypt + try (InputStream cipherInputStream = + GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), + secretKey, nonce, bcFipsProvider)) { + cipherTextOutput = readAllInputStream(cipherInputStream, GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length)); + } + //decrypt + try (InputStream plainInputStream = + GCMPacketsCipherInputStream.getDecryptor(new ByteArrayInputStream(cipherTextOutput.toByteArray()), + secretKey, nonce, bcFipsProvider)) { + plainTextOutput = readAllInputStream(plainInputStream, + GCMPacketsCipherInputStream.getDecryptionSizeFromCipherSize(cipherTextOutput.size())); + } + assertTrue(Arrays.equals(plainTextOutput.toByteArray(), 0, length, testPlaintextArray, startIndex, startIndex + length)); + } + + private SecretKey generateSecretKey() throws Exception { + return AccessController.doPrivileged((PrivilegedAction) () -> { + try { + KeyGenerator keyGen = KeyGenerator.getInstance("AES", bcFipsProvider); + keyGen.init(256, SecureRandom.getInstance("DEFAULT", bcFipsProvider)); + return keyGen.generateKey(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + // read "adversarily" in small random pieces + private ByteArrayOutputStream readAllInputStream(InputStream inputStream, long size) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(Math.toIntExact(size)); + byte[] temp = new byte[randomIntBetween(1, size != 0 ? Math.toIntExact(size) : 1)]; + do { + int bytesRead = inputStream.read(temp, 0, randomIntBetween(1, temp.length)); + if (bytesRead == -1) { + break; + } + baos.write(temp, 0, bytesRead); + if (randomBoolean()) { + int singleByte = inputStream.read(); + if (singleByte == -1) { + break; + } + baos.write(singleByte); + } + } while (true); + assertThat(baos.size(), Matchers.is(Math.toIntExact(size))); + return baos; + } +} From f78b5896c9a140da60b6e7567427a13f5b15fa3c Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 28 Nov 2019 23:39:26 +0200 Subject: [PATCH 29/29] Encryption new way --- .../encrypted/BufferOnMarkInputStream.java | 182 ++++++++++++++++++ .../encrypted/ChainInputStream.java | 146 ++++++++++++++ .../encrypted/CountingInputStream.java | 86 +++++++++ .../encrypted/DecryptionInputStream.java | 111 +++++++++++ .../encrypted/EncryptionInputStream.java | 168 ++++++++++++++++ .../encrypted/PrefixInputStream.java | 106 ++++++++++ 6 files changed, 799 insertions(+) create mode 100644 x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/BufferOnMarkInputStream.java create mode 100644 x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/ChainInputStream.java create mode 100644 x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/CountingInputStream.java create mode 100644 x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/DecryptionInputStream.java create mode 100644 x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptionInputStream.java create mode 100644 x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/PrefixInputStream.java diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/BufferOnMarkInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/BufferOnMarkInputStream.java new file mode 100644 index 0000000000000..2f364dd914e30 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/BufferOnMarkInputStream.java @@ -0,0 +1,182 @@ +package org.elasticsearch.repositories.encrypted; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Objects; + + +public final class BufferOnMarkInputStream extends FilterInputStream { + + private final int bufferSize; + private byte[] ringBuffer; + private int head; + private int tail; + private int position; + private boolean markCalled; + private boolean resetCalled; + private boolean closed; + + public BufferOnMarkInputStream(InputStream in, int bufferSize) { + super(Objects.requireNonNull(in)); + this.bufferSize = bufferSize; + this.ringBuffer = null; + this.head = this.tail = this.position = -1; + this.markCalled = this.resetCalled = false; + this.closed = false; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + ensureOpen(); + Objects.checkFromIndexSize(off, len, b.length); + if (len == 0) { + return 0; + } + if (resetCalled) { + int bytesRead = readFromBuffer(b, off, len); + if (bytesRead == 0) { + resetCalled = false; + } else { + return bytesRead; + } + } + int bytesRead = in.read(b, off, len); + if (bytesRead <= 0) { + return bytesRead; + } + if (markCalled) { + if (false == writeToBuffer(b, off, len)) { + // could not fully write to buffer, invalidate mark + markCalled = false; + } + } + return bytesRead; + } + + @Override + public int read() throws IOException { + ensureOpen(); + byte[] arr = new byte[1]; + int readResult = read(arr, 0, arr.length); + if (readResult == -1) { + return -1; + } + return arr[0]; + } + + @Override + public long skip(long n) throws IOException { + ensureOpen(); + if (n <= 0) { + return 0; + } + if (false == markCalled) { + return in.skip(n); + } + long remaining = n; + int size = (int)Math.min(2048, remaining); + byte[] skipBuffer = new byte[size]; + while (remaining > 0) { + int bytesRead = read(skipBuffer, 0, (int)Math.min(size, remaining)); + if (bytesRead < 0) { + break; + } + remaining -= bytesRead; + } + return n - remaining; + } + + @Override + public int available() throws IOException { + ensureOpen(); + int bytesAvailable = 0; + if (resetCalled) { + if (position < tail) { + bytesAvailable += tail - position; + } else { + bytesAvailable += ringBuffer.length - position + tail; + } + } + bytesAvailable += in.available(); + return bytesAvailable; + } + + @Override + public void mark(int readlimit) { + if (readlimit > bufferSize) { + throw new IllegalArgumentException("Readlimit value [" + readlimit + "] exceeds the maximum value of [" + bufferSize + "]"); + } + markCalled = true; + if (ringBuffer == null) { + ringBuffer = new byte[bufferSize]; + head = tail = position = 0; + } else { + head = position; + } + } + + @Override + public void reset() throws IOException { + ensureOpen(); + if (false == markCalled) { + throw new IOException("mark not called or has been invalidated"); + } + resetCalled = true; + } + + private int readFromBuffer(byte[] b, int off, int len) { + if (position == tail) { + return 0; + } + final int readLength; + if (position < tail) { + readLength = Math.min(len, tail - position); + } else { + readLength = Math.min(len, ringBuffer.length - position); + } + System.arraycopy(ringBuffer, position, b, off, readLength); + position += readLength; + if (position == ringBuffer.length) { + position = 0; + } + return readLength; + } + + private boolean writeToBuffer(byte[] b, int off, int len) { + while (len > 0 && head != tail) { + final int writeLength; + if (head < tail) { + writeLength = Math.min(len, ringBuffer.length - tail); + } else { + writeLength = Math.min(len, head - tail); + } + System.arraycopy(b, off, ringBuffer, tail, writeLength); + tail += writeLength; + off += writeLength; + len -= writeLength; + if (tail == ringBuffer.length) { + tail = 0; + } + } + if (len != 0) { + return false; + } + return true; + } + + private void ensureOpen() throws IOException { + if (closed) { + throw new IOException("Stream has been closed"); + } + } + + @Override + public void close() throws IOException { + if (false == closed) { + closed = true; + in.close(); + } + } + +} diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/ChainInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/ChainInputStream.java new file mode 100644 index 0000000000000..0618335a9f19b --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/ChainInputStream.java @@ -0,0 +1,146 @@ +package org.elasticsearch.repositories.encrypted; + +import org.elasticsearch.common.Nullable; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Objects; + +public abstract class ChainInputStream extends InputStream { + + private InputStream in; + private InputStream markIn; + private boolean closed; + + public ChainInputStream() { + this.in = null; + this.markIn = null; + this.closed = false; + } + + private void nextIn() throws IOException { + if (in != null) { + in.close(); + } + in = next(in); + if (in == null) { + throw new NullPointerException(); + } + if (markSupported() && false == in.markSupported()) { + throw new IllegalStateException("chain input stream element must support mark"); + } + } + + @Override + public int read() throws IOException { + ensureOpen(); + do { + int byteVal = in == null ? -1 : in.read(); + if (byteVal != -1) { + return byteVal; + } + if (false == hasNext(in)) { + return -1; + } + nextIn(); + } while (true); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + ensureOpen(); + Objects.checkFromIndexSize(off, len, b.length); + if (len == 0) { + return 0; + } + do { + int bytesRead = in == null ? -1 : in.read(b, off, len); + if (bytesRead != -1) { + return bytesRead; + } + if (false == hasNext(in)) { + return -1; + } + nextIn(); + } while (true); + } + + @Override + public long skip(long n) throws IOException { + ensureOpen(); + if (n <= 0) { + return 0; + } + long bytesRemaining = n; + while (bytesRemaining > 0) { + long bytesSkipped = in == null ? 0 : in.skip(bytesRemaining); + if (bytesSkipped == 0) { + int byteRead = read(); + if (byteRead == -1) { + break; + } else { + bytesRemaining--; + } + } else { + bytesRemaining -= bytesSkipped; + } + } + return n - bytesRemaining; + } + + @Override + public int available() throws IOException { + ensureOpen(); + return in == null ? 0 : in.available(); + } + + @Override + public boolean markSupported() { + return true; + } + + @Override + public void mark(int readlimit) { + if (markSupported()) { + markIn = in; + if (markIn != null) { + markIn.mark(readlimit); + } + } + } + + @Override + public void reset() throws IOException { + if (false == markSupported()) { + throw new IOException("Mark/reset not supported"); + } + in = markIn; + if (in != null) { + in.reset(); + } + } + + @Override + public void close() throws IOException { + if (false == closed) { + closed = true; + if (in != null) { + in.close(); + } + while (hasNext(in)) { + nextIn(); + } + } + } + + abstract boolean hasNext(@Nullable InputStream in); + + abstract InputStream next(@Nullable InputStream in) throws IOException; + + private void ensureOpen() throws IOException { + if (closed) { + throw new IOException("Stream is closed"); + } + } + +} diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/CountingInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/CountingInputStream.java new file mode 100644 index 0000000000000..fe1171e3cb95e --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/CountingInputStream.java @@ -0,0 +1,86 @@ +package org.elasticsearch.repositories.encrypted; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Objects; + +public final class CountingInputStream extends FilterInputStream { + + private long count; + private long mark; + private boolean closed; + private final boolean closeSource; + + /** + * Wraps another input stream, counting the number of bytes read. + * + * @param in the input stream to be wrapped + * @param closeSource if closing this stream will propagate to the wrapped stream + */ + public CountingInputStream(InputStream in, boolean closeSource) { + super(Objects.requireNonNull(in)); + this.count = 0L; + this.mark = -1L; + this.closed = false; + this.closeSource = closeSource; + } + + /** Returns the number of bytes read. */ + public long getCount() { + return count; + } + + @Override + public int read() throws IOException { + int result = in.read(); + if (result != -1) { + count++; + } + return result; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int result = in.read(b, off, len); + if (result != -1) { + count += result; + } + return result; + } + + @Override + public long skip(long n) throws IOException { + long result = in.skip(n); + count += result; + return result; + } + + @Override + public synchronized void mark(int readlimit) { + in.mark(readlimit); + mark = count; + } + + @Override + public synchronized void reset() throws IOException { + if (false == in.markSupported()) { + throw new IOException("Mark not supported"); + } + if (mark == -1L) { + throw new IOException("Mark not set"); + } + count = mark; + in.reset(); + } + + @Override + public void close() throws IOException { + if (false == closed) { + closed = true; + if (closeSource) { + in.close(); + } + } + } +} diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/DecryptionInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/DecryptionInputStream.java new file mode 100644 index 0000000000000..c1259355f7440 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/DecryptionInputStream.java @@ -0,0 +1,111 @@ +package org.elasticsearch.repositories.encrypted; + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; +import javax.crypto.ShortBufferException; +import javax.crypto.spec.GCMParameterSpec; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.util.NoSuchElementException; +import java.util.Objects; + +public class DecryptionInputStream extends ChainInputStream { + + private final InputStream source; + private final SecretKey secretKey; + private final int packetLength; + private final byte[] packet; + private final byte[] iv; + private boolean hasNext; + + public DecryptionInputStream(InputStream source, SecretKey secretKey, int packetLength) { + this.source = Objects.requireNonNull(source); + this.secretKey = Objects.requireNonNull(secretKey); + this.packetLength = packetLength; + this.packet = new byte[packetLength + EncryptedRepository.GCM_TAG_SIZE_IN_BYTES]; + this.iv = new byte[EncryptedRepository.GCM_IV_SIZE_IN_BYTES]; + this.hasNext = true; + } + + @Override + public boolean markSupported() { + return false; + } + + @Override + public void mark(int readlimit) { + } + + @Override + public void reset() throws IOException { + throw new IOException("Mark/reset not supported"); + } + + @Override + boolean hasNext(InputStream currentStream) { + return hasNext; + } + + private int decrypt(PrefixInputStream packetInputStream) throws IOException { + if (packetInputStream.read(iv) != iv.length) { + throw new IOException("Error while reading the heading IV of the packet"); + } + int packetLength = packetInputStream.read(packet); + if (packetLength < EncryptedRepository.GCM_TAG_SIZE_IN_BYTES) { + throw new IOException("Error while reading the packet"); + } + Cipher packetCipher = getPacketDecryptionCipher(iv); + try { + // in-place decryption + return packetCipher.doFinal(packet, 0, packetLength, packet); + } catch (ShortBufferException | IllegalBlockSizeException | BadPaddingException e) { + throw new IOException(e); + } + } + + private Cipher getPacketDecryptionCipher(byte[] packetIv) throws IOException { + GCMParameterSpec gcmParameterSpec = new GCMParameterSpec(EncryptedRepository.GCM_TAG_SIZE_IN_BYTES * Byte.SIZE, packetIv); + try { + Cipher packetCipher = Cipher.getInstance(EncryptedRepository.GCM_ENCRYPTION_SCHEME); + packetCipher.init(Cipher.DECRYPT_MODE, secretKey, gcmParameterSpec); + return packetCipher; + } catch (NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException | InvalidAlgorithmParameterException e) { + throw new IOException(e); + } + } + + @Override + InputStream next(InputStream currentStream) throws IOException { + if (currentStream != null && currentStream.read() != -1) { + throw new IllegalStateException("Stream for previous packet has not been fully processed"); + } + if (false == hasNext(currentStream)) { + throw new NoSuchElementException(); + } + PrefixInputStream packetInputStream = new PrefixInputStream(source, + packetLength + EncryptedRepository.GCM_IV_SIZE_IN_BYTES + EncryptedRepository.GCM_TAG_SIZE_IN_BYTES, + false); + int currentPacketLength = decrypt(packetInputStream); + if (currentPacketLength != packetLength) { + hasNext = false; + } + return new ByteArrayInputStream(packet, 0, currentPacketLength); + } + + public static long getDecryptionSize(long size, int packetLength) { + long encryptedPacketLength = packetLength + EncryptedRepository.GCM_TAG_SIZE_IN_BYTES + EncryptedRepository.GCM_IV_SIZE_IN_BYTES; + long completePackets = size / encryptedPacketLength; + long decryptedSize = completePackets * packetLength; + if (size % encryptedPacketLength != 0) { + decryptedSize += (size % encryptedPacketLength) - EncryptedRepository.GCM_TAG_SIZE_IN_BYTES - EncryptedRepository.GCM_TAG_SIZE_IN_BYTES; + } + return decryptedSize; + } +} diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptionInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptionInputStream.java new file mode 100644 index 0000000000000..076c5651ba452 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptionInputStream.java @@ -0,0 +1,168 @@ +package org.elasticsearch.repositories.encrypted; + +import org.elasticsearch.common.hash.MessageDigests; + +import javax.crypto.Cipher; +import javax.crypto.CipherInputStream; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; +import javax.crypto.spec.GCMParameterSpec; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.SequenceInputStream; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.util.HashSet; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Set; + +public final class EncryptionInputStream extends ChainInputStream { + + private final InputStream source; + private final SecretKey secretKey; + private final int packetLength; + private final int encryptedPacketLength; + + private IvGenerator currentIvGenerator; + private IvGenerator markIvGenerator; + private int markSourceOnNextPacket; + + public EncryptionInputStream(InputStream source, SecretKey secretKey, int packetLength) { + this.source = Objects.requireNonNull(source); + this.secretKey = Objects.requireNonNull(secretKey); + this.packetLength = packetLength; + this.encryptedPacketLength = packetLength + EncryptedRepository.GCM_IV_SIZE_IN_BYTES + EncryptedRepository.GCM_TAG_SIZE_IN_BYTES; + this.currentIvGenerator = new IvGenerator(); + this.markIvGenerator = null; + this.markSourceOnNextPacket = -1; + } + + @Override + public boolean markSupported() { + return source.markSupported(); + } + + @Override + public void mark(int readlimit) { + if (markSupported()) { + if (readlimit <= 0) { + throw new IllegalArgumentException("Mark readlimit must be a positive integer"); + } + super.mark(encryptedPacketLength); + markIvGenerator = new IvGenerator(this.currentIvGenerator); + markSourceOnNextPacket = readlimit; + } + } + + @Override + public void reset() throws IOException { + if (false == markSupported()) { + throw new IOException("Mark/reset not supported"); + } + if (markIvGenerator == null) { + throw new IOException("Mark no set"); + } + super.reset(); + currentIvGenerator = new IvGenerator(markIvGenerator); + if (markSourceOnNextPacket == -1) { + source.reset(); + } + } + + @Override + boolean hasNext(InputStream currentStream) { + if (currentStream != null && currentStream instanceof CountingInputStream == false) { + throw new IllegalStateException(); + } + if (((CountingInputStream) currentStream).getCount() > encryptedPacketLength) { + throw new IllegalStateException(); + } + return currentStream == null || ((CountingInputStream) currentStream).getCount() == encryptedPacketLength; + } + + @Override + InputStream next(InputStream currentStream) throws IOException { + if (currentStream != null && currentStream.read() != -1) { + throw new IllegalStateException("Stream for previous packet has not been fully processed"); + } + if (false == hasNext(currentStream)) { + throw new NoSuchElementException(); + } + if (markSourceOnNextPacket != -1) { + markSourceOnNextPacket = -1; + source.mark(markSourceOnNextPacket); + } + InputStream encryptionInputStream = new PrefixInputStream(source, packetLength, false); + byte[] packetIv = currentIvGenerator.newRandomUniqueIv(); + Cipher packetCipher = getPacketEncryptionCipher(packetIv); + encryptionInputStream = new CipherInputStream(encryptionInputStream, packetCipher); + encryptionInputStream = new SequenceInputStream(new ByteArrayInputStream(packetIv), encryptionInputStream); + encryptionInputStream = new BufferOnMarkInputStream(encryptionInputStream, packetLength); + return new CountingInputStream(encryptionInputStream, false); + } + + private Cipher getPacketEncryptionCipher(byte[] packetIv) throws IOException { + GCMParameterSpec gcmParameterSpec = new GCMParameterSpec(EncryptedRepository.GCM_TAG_SIZE_IN_BYTES * Byte.SIZE, packetIv); + try { + Cipher packetCipher = Cipher.getInstance(EncryptedRepository.GCM_ENCRYPTION_SCHEME); + packetCipher.init(Cipher.ENCRYPT_MODE, secretKey, gcmParameterSpec); + return packetCipher; + } catch (NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException | InvalidAlgorithmParameterException e) { + throw new IOException(e); + } + } + + public static long getEncryptionSize(long size, int packetLength) { + return size + (size / packetLength + 1) * (EncryptedRepository.GCM_TAG_SIZE_IN_BYTES + EncryptedRepository.GCM_IV_SIZE_IN_BYTES); + } + + static class IvGenerator { + + private final SecureRandom secureRandom; + private final Set previousIvs; + + IvGenerator() { + this.secureRandom = new SecureRandom(); + this.previousIvs = new HashSet<>(); + } + + IvGenerator(IvGenerator other) { + this(other.secureRandom, other.previousIvs); + } + + IvGenerator(SecureRandom secureRandom, Set previousIvs) { + try { + this.secureRandom = cloneRandom(secureRandom); + } catch (Exception e) { + throw new Error(e); + } + this.previousIvs = new HashSet<>(previousIvs); + } + + byte[] newRandomUniqueIv() { + byte[] iv = new byte[EncryptedRepository.GCM_TAG_SIZE_IN_BYTES]; + do { + secureRandom.nextBytes(iv); + } while (false == previousIvs.add(MessageDigests.toHexString(iv))); + return iv; + } + + private static SecureRandom cloneRandom(SecureRandom src) throws Exception { + ByteArrayOutputStream bo = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream(bo); + oos.writeObject(src); + oos.close(); + ObjectInputStream ois = new ObjectInputStream( + new ByteArrayInputStream(bo.toByteArray())); + return (SecureRandom)(ois.readObject()); + } + } + +} diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/PrefixInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/PrefixInputStream.java new file mode 100644 index 0000000000000..8cf280126d915 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/PrefixInputStream.java @@ -0,0 +1,106 @@ +package org.elasticsearch.repositories.encrypted; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Objects; + +public final class PrefixInputStream extends FilterInputStream { + + private final int length; + private int position; + private boolean closeSource; + private boolean closed; + + public PrefixInputStream(InputStream in, int length, boolean closeSource) { + super(Objects.requireNonNull(in)); + this.length = length; + this.position = 0; + this.closeSource = closeSource; + this.closed = false; + } + + @Override + public int read() throws IOException { + ensureOpen(); + if (position >= length) { + return -1; + } + int byteVal = in.read(); + if (byteVal == -1) { + return -1; + } + position++; + return byteVal; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + ensureOpen(); + Objects.checkFromIndexSize(off, len, b.length); + if (len == 0) { + return 0; + } + if (position >= length) { + return -1; + } + int readSize = Math.min(len, length - position); + int bytesRead = in.read(b, off, readSize); + if (bytesRead == -1) { + return -1; + } + position += bytesRead; + return bytesRead; + } + + @Override + public long skip(long n) throws IOException { + ensureOpen(); + if (n <= 0 || position >= length) { + return 0; + } + long bytesToSkip = Math.min(n, length - position); + assert bytesToSkip > 0; + long bytesSkipped = in.skip(bytesToSkip); + position += bytesSkipped; + return bytesSkipped; + } + + @Override + public int available() throws IOException { + ensureOpen(); + return Math.min(length - position, super.available()); + } + + @Override + public void mark(int readlimit) { + } + + @Override + public void reset() throws IOException { + throw new IOException("mark/reset not supported"); + } + + @Override + public boolean markSupported() { + return false; + } + + private void ensureOpen() throws IOException { + if (closed) { + throw new IOException("Stream has been closed"); + } + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + closed = true; + if (closeSource) { + in.close(); + } + } + +}