diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java index 7b7a9108c1ef5..b61466b2c1a51 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java @@ -124,7 +124,7 @@ protected AzureBlobStore createBlobStore() { } @Override - protected ByteSizeValue chunkSize() { + public ByteSizeValue chunkSize() { return chunkSize; } diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java index 852b147346e06..f0b516988baba 100644 --- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java +++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java @@ -94,7 +94,7 @@ protected GoogleCloudStorageBlobStore createBlobStore() { } @Override - protected ByteSizeValue chunkSize() { + public ByteSizeValue chunkSize() { return chunkSize; } diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java index 776c97422e5ee..b9aa10c0d440c 100644 --- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java @@ -233,7 +233,7 @@ protected HdfsBlobStore createBlobStore() { } @Override - protected ByteSizeValue chunkSize() { + public ByteSizeValue chunkSize() { return chunkSize; } } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index db6968ea71059..9ec7d54e9a186 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -207,7 +207,7 @@ protected BlobStore getBlobStore() { } @Override - protected ByteSizeValue chunkSize() { + public ByteSizeValue chunkSize() { return chunkSize; } } diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index ba670c2d6f01d..6c93628edd47f 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -200,6 +200,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp protected final ChecksumBlobStoreFormat snapshotFormat; + private final NamedXContentRegistry namedXContentRegistry; + private final boolean readOnly; private final ChecksumBlobStoreFormat indexShardSnapshotFormat; @@ -234,6 +236,7 @@ protected BlobStoreRepository( restoreRateLimiter = getRateLimiter(metadata.settings(), "max_restore_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB)); readOnly = metadata.settings().getAsBoolean("readonly", false); this.basePath = basePath; + this.namedXContentRegistry = namedXContentRegistry; indexShardSnapshotFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_CODEC, SNAPSHOT_NAME_FORMAT, BlobStoreIndexShardSnapshot::fromXContent, namedXContentRegistry, compress); diff --git a/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java b/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java index 9d69dea97f020..c477789c6e726 100644 --- a/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java @@ -109,7 +109,7 @@ protected BlobStore createBlobStore() throws Exception { } @Override - protected ByteSizeValue chunkSize() { + public ByteSizeValue chunkSize() { return chunkSize; } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java index 02fd885b3e350..1ce23725364e9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java @@ -76,7 +76,6 @@ import java.time.Clock; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -310,12 +309,6 @@ public static Path resolveConfigFile(Environment env, String name) { return config; } - @Override - public Map getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry, - ClusterService clusterService) { - return Collections.singletonMap("source", SourceOnlySnapshotRepository.newRepositoryFactory()); - } - @Override public Optional getEngineFactory(IndexSettings indexSettings) { if (indexSettings.getValue(SourceOnlySnapshotRepository.SOURCE_ONLY)) { diff --git a/x-pack/plugin/repository-encrypted/build.gradle b/x-pack/plugin/repository-encrypted/build.gradle new file mode 100644 index 0000000000000..3773140c02c64 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/build.gradle @@ -0,0 +1,31 @@ +evaluationDependsOn(xpackModule('core')) + +apply plugin: 'elasticsearch.esplugin' +esplugin { + name 'repository-encrypted' + description 'Elasticsearch Expanded Pack Plugin - client-side encrypted repositories.' + classname 'org.elasticsearch.repositories.encrypted.EncryptedRepositoryPlugin' + extendedPlugins = ['x-pack-core'] +} + +thirdPartyAudit { + ignoreViolations ( + 'org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider$CoreSecureRandom', + 'org.bouncycastle.jcajce.provider.ProvSunTLSKDF', + 'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$BaseTLSKeyGeneratorSpi', + 'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSKeyMaterialGenerator', + 'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSKeyMaterialGenerator$2', + 'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSMasterSecretGenerator', + 'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSMasterSecretGenerator$2', + 'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSPRFKeyGenerator', + 'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSRsaPreMasterSecretGenerator', + 'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSRsaPreMasterSecretGenerator$2', + ) +} + +dependencies { + compile "org.bouncycastle:bc-fips:1.0.1" + compile "org.bouncycastle:bcpkix-fips:1.0.3" +} + +integTest.enabled = false diff --git a/x-pack/plugin/repository-encrypted/licenses/bc-fips-1.0.1.jar.sha1 b/x-pack/plugin/repository-encrypted/licenses/bc-fips-1.0.1.jar.sha1 new file mode 100644 index 0000000000000..2e4bb227b43bc --- /dev/null +++ b/x-pack/plugin/repository-encrypted/licenses/bc-fips-1.0.1.jar.sha1 @@ -0,0 +1 @@ +ed8dd3144761eaa33b9c56f5e2bef85f1b731d6f \ No newline at end of file diff --git a/x-pack/plugin/repository-encrypted/licenses/bc-fips-LICENSE.txt b/x-pack/plugin/repository-encrypted/licenses/bc-fips-LICENSE.txt new file mode 100644 index 0000000000000..e94fe212ff725 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/licenses/bc-fips-LICENSE.txt @@ -0,0 +1,12 @@ +Copyright (c) 2000 - 2019 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/x-pack/plugin/repository-encrypted/licenses/bc-fips-NOTICE.txt b/x-pack/plugin/repository-encrypted/licenses/bc-fips-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/x-pack/plugin/repository-encrypted/licenses/bcpkix-fips-1.0.3.jar.sha1 b/x-pack/plugin/repository-encrypted/licenses/bcpkix-fips-1.0.3.jar.sha1 new file mode 100644 index 0000000000000..3262bdf0f3d03 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/licenses/bcpkix-fips-1.0.3.jar.sha1 @@ -0,0 +1 @@ +33c47b105777c9dcc8a08188186bd35401366bd1 \ No newline at end of file diff --git a/x-pack/plugin/repository-encrypted/licenses/bcpkix-fips-LICENSE.txt b/x-pack/plugin/repository-encrypted/licenses/bcpkix-fips-LICENSE.txt new file mode 100644 index 0000000000000..e94fe212ff725 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/licenses/bcpkix-fips-LICENSE.txt @@ -0,0 +1,12 @@ +Copyright (c) 2000 - 2019 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/x-pack/plugin/repository-encrypted/licenses/bcpkix-fips-NOTICE.txt b/x-pack/plugin/repository-encrypted/licenses/bcpkix-fips-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/BlobEncryptionMetadata.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/BlobEncryptionMetadata.java new file mode 100644 index 0000000000000..4dda3bff48f05 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/BlobEncryptionMetadata.java @@ -0,0 +1,180 @@ +package org.elasticsearch.repositories.encrypted; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +public class BlobEncryptionMetadata { + + private final int maxPacketSizeInBytes; + private final int authTagSizeInBytes; + private final int ivSizeInBytes; + private final byte[] dataEncryptionKeyMaterial; + private List packetsInfoList; + + public BlobEncryptionMetadata(int maxPacketSizeInBytes, int ivSizeInBytes, int authTagSizeInBytes, + byte[] dataEncryptionKeyMaterial, List packetsInfoList) { + this.maxPacketSizeInBytes = maxPacketSizeInBytes; + this.ivSizeInBytes = ivSizeInBytes; + this.authTagSizeInBytes = authTagSizeInBytes; + this.dataEncryptionKeyMaterial = dataEncryptionKeyMaterial; + // consistency check of the packet infos + for (PacketInfo packetInfo : packetsInfoList) { + if (packetInfo.getSizeInBytes() > maxPacketSizeInBytes) { + throw new IllegalArgumentException(); + } + if (packetInfo.getIv().length != ivSizeInBytes) { + throw new IllegalArgumentException(); + } + if (packetInfo.getAuthTag().length != authTagSizeInBytes) { + throw new IllegalArgumentException(); + } + } + this.packetsInfoList = Collections.unmodifiableList(packetsInfoList); + } + + public BlobEncryptionMetadata(InputStream inputStream) throws IOException { + this.maxPacketSizeInBytes = readInt(inputStream); + this.authTagSizeInBytes = readInt(inputStream); + this.ivSizeInBytes = readInt(inputStream); + + int dataEncryptionKeySizeInBytes = readInt(inputStream); + this.dataEncryptionKeyMaterial = readExactlyNBytes(inputStream, dataEncryptionKeySizeInBytes); + + int packetsInfoListSize = readInt(inputStream); + List packetsInfo = new ArrayList<>(packetsInfoListSize); + for (int i = 0; i < packetsInfoListSize; i++) { + PacketInfo packetInfo = new PacketInfo(inputStream, ivSizeInBytes, authTagSizeInBytes); + // consistency check of the packet infos + if (packetInfo.getSizeInBytes() > this.maxPacketSizeInBytes) { + throw new IllegalArgumentException(); + } + if (packetInfo.getIv().length != this.ivSizeInBytes) { + throw new IllegalArgumentException(); + } + if (packetInfo.getAuthTag().length != this.authTagSizeInBytes) { + throw new IllegalArgumentException(); + } + packetsInfo.add(packetInfo); + } + this.packetsInfoList = Collections.unmodifiableList(packetsInfo); + } + + public byte[] getDataEncryptionKeyMaterial() { + return dataEncryptionKeyMaterial; + } + + public int getMaxPacketSizeInBytes() { + return maxPacketSizeInBytes; + } + + public int getAuthTagSizeInBytes() { + return authTagSizeInBytes; + } + + public int getIvSizeInBytes() { + return ivSizeInBytes; + } + + public List getPacketsInfoList() { + return packetsInfoList; + } + + public void write(OutputStream out) throws IOException { + writeInt(out, maxPacketSizeInBytes); + writeInt(out, authTagSizeInBytes); + writeInt(out, ivSizeInBytes); + + writeInt(out, dataEncryptionKeyMaterial.length); + out.write(dataEncryptionKeyMaterial); + + writeInt(out, packetsInfoList.size()); + for (PacketInfo packetInfo : packetsInfoList) { + packetInfo.write(out); + } + } + + private static int readInt(InputStream inputStream) throws IOException { + return ((inputStream.read() & 0xFF) << 24) | + ((inputStream.read() & 0xFF) << 16) | + ((inputStream.read() & 0xFF) << 8 ) | + ((inputStream.read() & 0xFF) << 0 ); + } + + private static void writeInt(OutputStream out, int val) throws IOException { + out.write(val >>> 24); + out.write(val >>> 16); + out.write(val >>> 8); + out.write(val); + } + + private static byte[] readExactlyNBytes(InputStream inputStream, int nBytes) throws IOException { + byte[] ans = new byte[nBytes]; + if (nBytes != inputStream.readNBytes(ans, 0, nBytes)) { + throw new IOException("Fewer than [" + nBytes + "] read"); + } + return ans; + } + + static class PacketInfo { + + private final byte[] iv; + private final byte[] authTag; + private final int sizeInBytes; + + PacketInfo(byte[] iv, byte[] authTag, int sizeInBytes) { + this.iv = iv; + this.authTag = authTag; + this.sizeInBytes = sizeInBytes; + } + + PacketInfo(InputStream inputStream, int ivSizeInBytes, int authTagSizeInBytes) throws IOException { + this.iv = readExactlyNBytes(inputStream, ivSizeInBytes); + this.authTag = readExactlyNBytes(inputStream, authTagSizeInBytes); + this.sizeInBytes = readInt(inputStream); + } + + byte[] getIv() { + return iv; + } + + byte[] getAuthTag() { + return authTag; + } + + int getSizeInBytes() { + return sizeInBytes; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PacketInfo that = (PacketInfo) o; + return sizeInBytes == that.sizeInBytes && + Arrays.equals(iv, that.iv) && + Arrays.equals(authTag, that.authTag); + } + + @Override + public int hashCode() { + int result = Objects.hash(sizeInBytes); + result = 31 * result + Arrays.hashCode(iv); + result = 31 * result + Arrays.hashCode(authTag); + return result; + } + + public void write(OutputStream out) throws IOException { + out.write(iv); + out.write(authTag); + writeInt(out, sizeInBytes); + } + + } + +} diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/BufferOnMarkInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/BufferOnMarkInputStream.java new file mode 100644 index 0000000000000..2f364dd914e30 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/BufferOnMarkInputStream.java @@ -0,0 +1,182 @@ +package org.elasticsearch.repositories.encrypted; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Objects; + + +public final class BufferOnMarkInputStream extends FilterInputStream { + + private final int bufferSize; + private byte[] ringBuffer; + private int head; + private int tail; + private int position; + private boolean markCalled; + private boolean resetCalled; + private boolean closed; + + public BufferOnMarkInputStream(InputStream in, int bufferSize) { + super(Objects.requireNonNull(in)); + this.bufferSize = bufferSize; + this.ringBuffer = null; + this.head = this.tail = this.position = -1; + this.markCalled = this.resetCalled = false; + this.closed = false; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + ensureOpen(); + Objects.checkFromIndexSize(off, len, b.length); + if (len == 0) { + return 0; + } + if (resetCalled) { + int bytesRead = readFromBuffer(b, off, len); + if (bytesRead == 0) { + resetCalled = false; + } else { + return bytesRead; + } + } + int bytesRead = in.read(b, off, len); + if (bytesRead <= 0) { + return bytesRead; + } + if (markCalled) { + if (false == writeToBuffer(b, off, len)) { + // could not fully write to buffer, invalidate mark + markCalled = false; + } + } + return bytesRead; + } + + @Override + public int read() throws IOException { + ensureOpen(); + byte[] arr = new byte[1]; + int readResult = read(arr, 0, arr.length); + if (readResult == -1) { + return -1; + } + return arr[0]; + } + + @Override + public long skip(long n) throws IOException { + ensureOpen(); + if (n <= 0) { + return 0; + } + if (false == markCalled) { + return in.skip(n); + } + long remaining = n; + int size = (int)Math.min(2048, remaining); + byte[] skipBuffer = new byte[size]; + while (remaining > 0) { + int bytesRead = read(skipBuffer, 0, (int)Math.min(size, remaining)); + if (bytesRead < 0) { + break; + } + remaining -= bytesRead; + } + return n - remaining; + } + + @Override + public int available() throws IOException { + ensureOpen(); + int bytesAvailable = 0; + if (resetCalled) { + if (position < tail) { + bytesAvailable += tail - position; + } else { + bytesAvailable += ringBuffer.length - position + tail; + } + } + bytesAvailable += in.available(); + return bytesAvailable; + } + + @Override + public void mark(int readlimit) { + if (readlimit > bufferSize) { + throw new IllegalArgumentException("Readlimit value [" + readlimit + "] exceeds the maximum value of [" + bufferSize + "]"); + } + markCalled = true; + if (ringBuffer == null) { + ringBuffer = new byte[bufferSize]; + head = tail = position = 0; + } else { + head = position; + } + } + + @Override + public void reset() throws IOException { + ensureOpen(); + if (false == markCalled) { + throw new IOException("mark not called or has been invalidated"); + } + resetCalled = true; + } + + private int readFromBuffer(byte[] b, int off, int len) { + if (position == tail) { + return 0; + } + final int readLength; + if (position < tail) { + readLength = Math.min(len, tail - position); + } else { + readLength = Math.min(len, ringBuffer.length - position); + } + System.arraycopy(ringBuffer, position, b, off, readLength); + position += readLength; + if (position == ringBuffer.length) { + position = 0; + } + return readLength; + } + + private boolean writeToBuffer(byte[] b, int off, int len) { + while (len > 0 && head != tail) { + final int writeLength; + if (head < tail) { + writeLength = Math.min(len, ringBuffer.length - tail); + } else { + writeLength = Math.min(len, head - tail); + } + System.arraycopy(b, off, ringBuffer, tail, writeLength); + tail += writeLength; + off += writeLength; + len -= writeLength; + if (tail == ringBuffer.length) { + tail = 0; + } + } + if (len != 0) { + return false; + } + return true; + } + + private void ensureOpen() throws IOException { + if (closed) { + throw new IOException("Stream has been closed"); + } + } + + @Override + public void close() throws IOException { + if (false == closed) { + closed = true; + in.close(); + } + } + +} diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/ChainInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/ChainInputStream.java new file mode 100644 index 0000000000000..0618335a9f19b --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/ChainInputStream.java @@ -0,0 +1,146 @@ +package org.elasticsearch.repositories.encrypted; + +import org.elasticsearch.common.Nullable; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Objects; + +public abstract class ChainInputStream extends InputStream { + + private InputStream in; + private InputStream markIn; + private boolean closed; + + public ChainInputStream() { + this.in = null; + this.markIn = null; + this.closed = false; + } + + private void nextIn() throws IOException { + if (in != null) { + in.close(); + } + in = next(in); + if (in == null) { + throw new NullPointerException(); + } + if (markSupported() && false == in.markSupported()) { + throw new IllegalStateException("chain input stream element must support mark"); + } + } + + @Override + public int read() throws IOException { + ensureOpen(); + do { + int byteVal = in == null ? -1 : in.read(); + if (byteVal != -1) { + return byteVal; + } + if (false == hasNext(in)) { + return -1; + } + nextIn(); + } while (true); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + ensureOpen(); + Objects.checkFromIndexSize(off, len, b.length); + if (len == 0) { + return 0; + } + do { + int bytesRead = in == null ? -1 : in.read(b, off, len); + if (bytesRead != -1) { + return bytesRead; + } + if (false == hasNext(in)) { + return -1; + } + nextIn(); + } while (true); + } + + @Override + public long skip(long n) throws IOException { + ensureOpen(); + if (n <= 0) { + return 0; + } + long bytesRemaining = n; + while (bytesRemaining > 0) { + long bytesSkipped = in == null ? 0 : in.skip(bytesRemaining); + if (bytesSkipped == 0) { + int byteRead = read(); + if (byteRead == -1) { + break; + } else { + bytesRemaining--; + } + } else { + bytesRemaining -= bytesSkipped; + } + } + return n - bytesRemaining; + } + + @Override + public int available() throws IOException { + ensureOpen(); + return in == null ? 0 : in.available(); + } + + @Override + public boolean markSupported() { + return true; + } + + @Override + public void mark(int readlimit) { + if (markSupported()) { + markIn = in; + if (markIn != null) { + markIn.mark(readlimit); + } + } + } + + @Override + public void reset() throws IOException { + if (false == markSupported()) { + throw new IOException("Mark/reset not supported"); + } + in = markIn; + if (in != null) { + in.reset(); + } + } + + @Override + public void close() throws IOException { + if (false == closed) { + closed = true; + if (in != null) { + in.close(); + } + while (hasNext(in)) { + nextIn(); + } + } + } + + abstract boolean hasNext(@Nullable InputStream in); + + abstract InputStream next(@Nullable InputStream in) throws IOException; + + private void ensureOpen() throws IOException { + if (closed) { + throw new IOException("Stream is closed"); + } + } + +} diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/CountingInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/CountingInputStream.java new file mode 100644 index 0000000000000..fe1171e3cb95e --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/CountingInputStream.java @@ -0,0 +1,86 @@ +package org.elasticsearch.repositories.encrypted; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Objects; + +public final class CountingInputStream extends FilterInputStream { + + private long count; + private long mark; + private boolean closed; + private final boolean closeSource; + + /** + * Wraps another input stream, counting the number of bytes read. + * + * @param in the input stream to be wrapped + * @param closeSource if closing this stream will propagate to the wrapped stream + */ + public CountingInputStream(InputStream in, boolean closeSource) { + super(Objects.requireNonNull(in)); + this.count = 0L; + this.mark = -1L; + this.closed = false; + this.closeSource = closeSource; + } + + /** Returns the number of bytes read. */ + public long getCount() { + return count; + } + + @Override + public int read() throws IOException { + int result = in.read(); + if (result != -1) { + count++; + } + return result; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int result = in.read(b, off, len); + if (result != -1) { + count += result; + } + return result; + } + + @Override + public long skip(long n) throws IOException { + long result = in.skip(n); + count += result; + return result; + } + + @Override + public synchronized void mark(int readlimit) { + in.mark(readlimit); + mark = count; + } + + @Override + public synchronized void reset() throws IOException { + if (false == in.markSupported()) { + throw new IOException("Mark not supported"); + } + if (mark == -1L) { + throw new IOException("Mark not set"); + } + count = mark; + in.reset(); + } + + @Override + public void close() throws IOException { + if (false == closed) { + closed = true; + if (closeSource) { + in.close(); + } + } + } +} diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/DecryptionInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/DecryptionInputStream.java new file mode 100644 index 0000000000000..c1259355f7440 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/DecryptionInputStream.java @@ -0,0 +1,111 @@ +package org.elasticsearch.repositories.encrypted; + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; +import javax.crypto.ShortBufferException; +import javax.crypto.spec.GCMParameterSpec; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.util.NoSuchElementException; +import java.util.Objects; + +public class DecryptionInputStream extends ChainInputStream { + + private final InputStream source; + private final SecretKey secretKey; + private final int packetLength; + private final byte[] packet; + private final byte[] iv; + private boolean hasNext; + + public DecryptionInputStream(InputStream source, SecretKey secretKey, int packetLength) { + this.source = Objects.requireNonNull(source); + this.secretKey = Objects.requireNonNull(secretKey); + this.packetLength = packetLength; + this.packet = new byte[packetLength + EncryptedRepository.GCM_TAG_SIZE_IN_BYTES]; + this.iv = new byte[EncryptedRepository.GCM_IV_SIZE_IN_BYTES]; + this.hasNext = true; + } + + @Override + public boolean markSupported() { + return false; + } + + @Override + public void mark(int readlimit) { + } + + @Override + public void reset() throws IOException { + throw new IOException("Mark/reset not supported"); + } + + @Override + boolean hasNext(InputStream currentStream) { + return hasNext; + } + + private int decrypt(PrefixInputStream packetInputStream) throws IOException { + if (packetInputStream.read(iv) != iv.length) { + throw new IOException("Error while reading the heading IV of the packet"); + } + int packetLength = packetInputStream.read(packet); + if (packetLength < EncryptedRepository.GCM_TAG_SIZE_IN_BYTES) { + throw new IOException("Error while reading the packet"); + } + Cipher packetCipher = getPacketDecryptionCipher(iv); + try { + // in-place decryption + return packetCipher.doFinal(packet, 0, packetLength, packet); + } catch (ShortBufferException | IllegalBlockSizeException | BadPaddingException e) { + throw new IOException(e); + } + } + + private Cipher getPacketDecryptionCipher(byte[] packetIv) throws IOException { + GCMParameterSpec gcmParameterSpec = new GCMParameterSpec(EncryptedRepository.GCM_TAG_SIZE_IN_BYTES * Byte.SIZE, packetIv); + try { + Cipher packetCipher = Cipher.getInstance(EncryptedRepository.GCM_ENCRYPTION_SCHEME); + packetCipher.init(Cipher.DECRYPT_MODE, secretKey, gcmParameterSpec); + return packetCipher; + } catch (NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException | InvalidAlgorithmParameterException e) { + throw new IOException(e); + } + } + + @Override + InputStream next(InputStream currentStream) throws IOException { + if (currentStream != null && currentStream.read() != -1) { + throw new IllegalStateException("Stream for previous packet has not been fully processed"); + } + if (false == hasNext(currentStream)) { + throw new NoSuchElementException(); + } + PrefixInputStream packetInputStream = new PrefixInputStream(source, + packetLength + EncryptedRepository.GCM_IV_SIZE_IN_BYTES + EncryptedRepository.GCM_TAG_SIZE_IN_BYTES, + false); + int currentPacketLength = decrypt(packetInputStream); + if (currentPacketLength != packetLength) { + hasNext = false; + } + return new ByteArrayInputStream(packet, 0, currentPacketLength); + } + + public static long getDecryptionSize(long size, int packetLength) { + long encryptedPacketLength = packetLength + EncryptedRepository.GCM_TAG_SIZE_IN_BYTES + EncryptedRepository.GCM_IV_SIZE_IN_BYTES; + long completePackets = size / encryptedPacketLength; + long decryptedSize = completePackets * packetLength; + if (size % encryptedPacketLength != 0) { + decryptedSize += (size % encryptedPacketLength) - EncryptedRepository.GCM_TAG_SIZE_IN_BYTES - EncryptedRepository.GCM_TAG_SIZE_IN_BYTES; + } + return decryptedSize; + } +} diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java new file mode 100644 index 0000000000000..3400f7a169ee4 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java @@ -0,0 +1,317 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.repositories.encrypted; + +import org.bouncycastle.asn1.ASN1ObjectIdentifier; +import org.bouncycastle.asn1.cms.CMSObjectIdentifiers; +import org.bouncycastle.cms.CMSAlgorithm; +import org.bouncycastle.cms.CMSEnvelopedData; +import org.bouncycastle.cms.CMSEnvelopedDataGenerator; +import org.bouncycastle.cms.CMSException; +import org.bouncycastle.cms.CMSTypedData; +import org.bouncycastle.cms.PasswordRecipientInfoGenerator; +import org.bouncycastle.cms.PasswordRecipientInformation; +import org.bouncycastle.cms.RecipientInformation; +import org.bouncycastle.cms.RecipientInformationStore; +import org.bouncycastle.cms.jcajce.JceCMSContentEncryptorBuilder; +import org.bouncycastle.cms.jcajce.JcePasswordEnvelopedRecipient; +import org.bouncycastle.cms.jcajce.JcePasswordRecipientInfoGenerator; +import org.elasticsearch.cluster.metadata.RepositoryMetaData; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.blobstore.BlobContainer; +import org.elasticsearch.common.blobstore.BlobMetaData; +import org.elasticsearch.common.blobstore.BlobPath; +import org.elasticsearch.common.blobstore.BlobStore; +import org.elasticsearch.common.blobstore.DeleteResult; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.settings.SecureSetting; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.blobstore.BlobStoreRepository; + +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +public class EncryptedRepository extends BlobStoreRepository { + + static final int ENCRYPTION_PROTOCOL_VERSION_NUMBER = 1; + // this is chosen somewhat arbitrarily. Assuming no mark calls during encryption/upload/snapshot, a larger value increases + // the metadata/data size ratio, thereby reducing the encryption overhead, but it also requires that decryption/download/restore + // allocate and use a buffer of this size. + static final int MAX_PACKET_SIZE = 64 * 1024; // 64KB packet sizes + static final int GCM_TAG_SIZE_IN_BYTES = 16; + static final int GCM_IV_SIZE_IN_BYTES = 12; + static final String GCM_ENCRYPTION_SCHEME = "AES/GCM/NoPadding"; + static final int AES_BLOCK_SIZE_IN_BYTES = 128; + + static final Setting.AffixSetting ENCRYPTION_PASSWORD_SETTING = Setting.affixKeySetting("repository.encrypted.", + "password", key -> SecureSetting.secureString(key, null)); + + private static final Setting DELEGATE_TYPE = new Setting<>("delegate_type", "", Function.identity()); + private static final String ENCRYPTION_METADATA_PREFIX = "encryption-metadata-"; + + private final BlobStoreRepository delegatedRepository; + private final char[] masterPassword; + + protected EncryptedRepository(BlobStoreRepository delegatedRepository, char[] masterPassword) { + super(delegatedRepository); + this.delegatedRepository = delegatedRepository; + this.masterPassword = masterPassword; + } + + @Override + protected BlobStore createBlobStore() throws Exception { + return new EncryptedBlobStoreDecorator(this.delegatedRepository.blobStore(), masterPassword); + } + + @Override + protected void doStart() { + this.delegatedRepository.start(); + super.doStart(); + } + + @Override + protected void doStop() { + super.doStop(); + this.delegatedRepository.stop(); + } + + @Override + protected void doClose() { + super.doClose(); + this.delegatedRepository.close(); + } + + /** + * Returns a new encrypted repository factory + */ + public static Repository.Factory newRepositoryFactory(final Settings settings) { + final Map cachedRepositoryPasswords = new HashMap<>(); + for (String repositoryName : ENCRYPTION_PASSWORD_SETTING.getNamespaces(settings)) { + Setting encryptionPasswordSetting = ENCRYPTION_PASSWORD_SETTING + .getConcreteSettingForNamespace(repositoryName); + SecureString encryptionPassword = encryptionPasswordSetting.get(settings); + cachedRepositoryPasswords.put(repositoryName, encryptionPassword.getChars()); + } + return new Repository.Factory() { + + @Override + public Repository create(RepositoryMetaData metadata) { + throw new UnsupportedOperationException(); + } + + @Override + public Repository create(RepositoryMetaData metaData, Function typeLookup) throws Exception { + String delegateType = DELEGATE_TYPE.get(metaData.settings()); + if (Strings.hasLength(delegateType) == false) { + throw new IllegalArgumentException(DELEGATE_TYPE.getKey() + " must be set"); + } + + if (false == cachedRepositoryPasswords.containsKey(metaData.name())) { + throw new IllegalArgumentException( + ENCRYPTION_PASSWORD_SETTING.getConcreteSettingForNamespace(metaData.name()).getKey() + " must be set"); + } + Repository.Factory factory = typeLookup.apply(delegateType); + Repository delegatedRepository = factory.create(new RepositoryMetaData(metaData.name(), + delegateType, metaData.settings())); + if (false == (delegatedRepository instanceof BlobStoreRepository)) { + throw new IllegalArgumentException("Unsupported type " + DELEGATE_TYPE.getKey()); + } + char[] masterPassword = cachedRepositoryPasswords.get(metaData.name()); + return new EncryptedRepository((BlobStoreRepository) delegatedRepository, masterPassword); + } + }; + } + + private static class EncryptedBlobStoreDecorator implements BlobStore { + + private final BlobStore delegatedBlobStore; + private final char[] masterPassword; + + EncryptedBlobStoreDecorator(BlobStore blobStore, char[] masterPassword) { + this.delegatedBlobStore = blobStore; + this.masterPassword = masterPassword; + } + + @Override + public void close() throws IOException { + this.delegatedBlobStore.close(); + } + + @Override + public BlobContainer blobContainer(BlobPath path) { + BlobPath encryptionMetadataBlobPath = BlobPath.cleanPath(); + encryptionMetadataBlobPath = encryptionMetadataBlobPath.add(ENCRYPTION_METADATA_PREFIX + ENCRYPTION_PROTOCOL_VERSION_NUMBER); + for (String pathComponent : path) { + encryptionMetadataBlobPath = encryptionMetadataBlobPath.add(pathComponent); + } + return new EncryptedBlobContainerDecorator(this.delegatedBlobStore.blobContainer(path), + this.delegatedBlobStore.blobContainer(encryptionMetadataBlobPath), this.masterPassword); + } + } + + private static class EncryptedBlobContainerDecorator implements BlobContainer { + + private final BlobContainer delegatedBlobContainer; + private final BlobContainer encryptionMetadataBlobContainer; + private final char[] masterPassword; + + EncryptedBlobContainerDecorator(BlobContainer delegatedBlobContainer, BlobContainer encryptionMetadataBlobContainer, + char[] masterPassword) { + this.delegatedBlobContainer = delegatedBlobContainer; + this.encryptionMetadataBlobContainer = encryptionMetadataBlobContainer; + this.masterPassword = masterPassword; + } + + @Override + public BlobPath path() { + return this.delegatedBlobContainer.path(); + } + + @Override + public InputStream readBlob(String blobName) throws IOException { + BytesReference encryptedMetadataBytes = Streams.readFully(this.encryptionMetadataBlobContainer.readBlob(blobName)); + BlobEncryptionMetadata metadata = decryptMetadata(BytesReference.toBytes(encryptedMetadataBytes)); + SecretKey dataEncryptionKey = new SecretKeySpec(metadata.getDataEncryptionKeyMaterial(), 0, + metadata.getDataEncryptionKeyMaterial().length, "AES"); + return new GCMPacketsDecryptorInputStream(this.delegatedBlobContainer.readBlob(blobName), dataEncryptionKey, + metadata.getMaxPacketSizeInBytes(), metadata.getAuthTagSizeInBytes(), metadata.getPacketsInfoList().iterator()); + } + + @Override + public void writeBlob(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) throws IOException { + final SecretKey dataEncryptionKey; + try { + dataEncryptionKey = generateRandomSecretKey(); + } catch (NoSuchAlgorithmException e) { + throw new IOException(e); + } + GCMPacketsEncryptorInputStream encryptedInputStream = new GCMPacketsEncryptorInputStream(inputStream, dataEncryptionKey, + MAX_PACKET_SIZE); + try { + this.delegatedBlobContainer.writeBlob(blobName, encryptedInputStream, blobSize, failIfAlreadyExists); + } finally { + encryptedInputStream.close(); + } + List packetsMetadataList = encryptedInputStream.getEncryptionPacketMetadata(); + BlobEncryptionMetadata metadata = new BlobEncryptionMetadata(MAX_PACKET_SIZE, GCM_IV_SIZE_IN_BYTES, GCM_TAG_SIZE_IN_BYTES, + dataEncryptionKey.getEncoded(), packetsMetadataList); + byte[] encryptedMetadata = encryptMetadata(metadata); + try (InputStream stream = new ByteArrayInputStream(encryptedMetadata)) { + this.encryptionMetadataBlobContainer.writeBlob(blobName, stream, encryptedMetadata.length, false); + } + } + + private byte[] encryptMetadata(BlobEncryptionMetadata metadata) throws IOException { + CMSEnvelopedDataGenerator envelopedDataGenerator = new CMSEnvelopedDataGenerator(); + PasswordRecipientInfoGenerator passwordRecipientInfoGenerator = new JcePasswordRecipientInfoGenerator(CMSAlgorithm.AES256_GCM + , masterPassword); + envelopedDataGenerator.addRecipientInfoGenerator(passwordRecipientInfoGenerator); + final CMSEnvelopedData envelopedData; + try { + envelopedData = envelopedDataGenerator.generate(new CMSTypedData() { + @Override + public ASN1ObjectIdentifier getContentType() { + return CMSObjectIdentifiers.data; + } + + @Override + public void write(OutputStream out) throws IOException, CMSException { + metadata.write(out); + } + + @Override + public Object getContent() { + return metadata; + } + }, new JceCMSContentEncryptorBuilder(CMSAlgorithm.AES256_GCM).build()); + } catch (CMSException e) { + throw new IOException(e); + } + return envelopedData.getEncoded(); + } + + private BlobEncryptionMetadata decryptMetadata(byte[] metadata) throws IOException { + final CMSEnvelopedData envelopedData; + try { + envelopedData = new CMSEnvelopedData(metadata); + } catch (CMSException e) { + throw new IOException(e); + } + RecipientInformationStore recipients = envelopedData.getRecipientInfos(); + if (recipients.getRecipients().size() != 1) { + throw new IllegalStateException(); + } + RecipientInformation recipient = recipients.iterator().next(); + if (false == (recipient instanceof PasswordRecipientInformation)) { + throw new IllegalStateException(); + } + final byte[] decryptedMetadata; + try { + decryptedMetadata = recipient.getContent(new JcePasswordEnvelopedRecipient(masterPassword)); + } catch (CMSException e) { + throw new IOException(e); + } + return new BlobEncryptionMetadata(new ByteArrayInputStream(decryptedMetadata)); + } + + @Override + public void writeBlobAtomic(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) + throws IOException { + // does not support atomic write + writeBlob(blobName, inputStream, blobSize, failIfAlreadyExists); + } + + @Override + public void deleteBlob(String blobName) throws IOException { + this.encryptionMetadataBlobContainer.deleteBlob(blobName); + this.delegatedBlobContainer.deleteBlob(blobName); + } + + @Override + public DeleteResult delete() throws IOException { + this.encryptionMetadataBlobContainer.delete(); + return this.delegatedBlobContainer.delete(); + } + + @Override + public Map listBlobs() throws IOException { + return this.delegatedBlobContainer.listBlobs(); + } + + @Override + public Map children() throws IOException { + return this.delegatedBlobContainer.children(); + } + + @Override + public Map listBlobsByPrefix(String blobNamePrefix) throws IOException { + return this.delegatedBlobContainer.listBlobsByPrefix(blobNamePrefix); + } + } + + private static SecretKey generateRandomSecretKey() throws NoSuchAlgorithmException { + KeyGenerator keyGen = KeyGenerator.getInstance("AES"); + keyGen.init(256, SecureRandom.getInstance("DEFAULT")); + return keyGen.generateKey(); + } + +} diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepositoryPlugin.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepositoryPlugin.java new file mode 100644 index 0000000000000..c2631cdafac18 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepositoryPlugin.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.repositories.encrypted; + +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.ReloadablePlugin; +import org.elasticsearch.plugins.RepositoryPlugin; +import org.elasticsearch.repositories.Repository; +import org.elasticsearch.threadpool.ThreadPool; + +import java.util.List; +import java.util.Map; + +public class EncryptedRepositoryPlugin extends Plugin implements RepositoryPlugin, ReloadablePlugin { + + private final Repository.Factory encryptedRepositoryFactory; + + public EncryptedRepositoryPlugin(final Settings settings) { + encryptedRepositoryFactory = EncryptedRepository.newRepositoryFactory(settings); + } + + @Override + public Map getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry, + ThreadPool threadPool) { + return Map.of("encrypted", encryptedRepositoryFactory); + } + + @Override + public List> getSettings() { + return List.of(EncryptedRepository.ENCRYPTION_PASSWORD_SETTING); + } + + @Override + public void reload(Settings settings) { + // Secure settings should be readable inside this method. + } +} diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptionInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptionInputStream.java new file mode 100644 index 0000000000000..076c5651ba452 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptionInputStream.java @@ -0,0 +1,168 @@ +package org.elasticsearch.repositories.encrypted; + +import org.elasticsearch.common.hash.MessageDigests; + +import javax.crypto.Cipher; +import javax.crypto.CipherInputStream; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; +import javax.crypto.spec.GCMParameterSpec; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.SequenceInputStream; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.util.HashSet; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Set; + +public final class EncryptionInputStream extends ChainInputStream { + + private final InputStream source; + private final SecretKey secretKey; + private final int packetLength; + private final int encryptedPacketLength; + + private IvGenerator currentIvGenerator; + private IvGenerator markIvGenerator; + private int markSourceOnNextPacket; + + public EncryptionInputStream(InputStream source, SecretKey secretKey, int packetLength) { + this.source = Objects.requireNonNull(source); + this.secretKey = Objects.requireNonNull(secretKey); + this.packetLength = packetLength; + this.encryptedPacketLength = packetLength + EncryptedRepository.GCM_IV_SIZE_IN_BYTES + EncryptedRepository.GCM_TAG_SIZE_IN_BYTES; + this.currentIvGenerator = new IvGenerator(); + this.markIvGenerator = null; + this.markSourceOnNextPacket = -1; + } + + @Override + public boolean markSupported() { + return source.markSupported(); + } + + @Override + public void mark(int readlimit) { + if (markSupported()) { + if (readlimit <= 0) { + throw new IllegalArgumentException("Mark readlimit must be a positive integer"); + } + super.mark(encryptedPacketLength); + markIvGenerator = new IvGenerator(this.currentIvGenerator); + markSourceOnNextPacket = readlimit; + } + } + + @Override + public void reset() throws IOException { + if (false == markSupported()) { + throw new IOException("Mark/reset not supported"); + } + if (markIvGenerator == null) { + throw new IOException("Mark no set"); + } + super.reset(); + currentIvGenerator = new IvGenerator(markIvGenerator); + if (markSourceOnNextPacket == -1) { + source.reset(); + } + } + + @Override + boolean hasNext(InputStream currentStream) { + if (currentStream != null && currentStream instanceof CountingInputStream == false) { + throw new IllegalStateException(); + } + if (((CountingInputStream) currentStream).getCount() > encryptedPacketLength) { + throw new IllegalStateException(); + } + return currentStream == null || ((CountingInputStream) currentStream).getCount() == encryptedPacketLength; + } + + @Override + InputStream next(InputStream currentStream) throws IOException { + if (currentStream != null && currentStream.read() != -1) { + throw new IllegalStateException("Stream for previous packet has not been fully processed"); + } + if (false == hasNext(currentStream)) { + throw new NoSuchElementException(); + } + if (markSourceOnNextPacket != -1) { + markSourceOnNextPacket = -1; + source.mark(markSourceOnNextPacket); + } + InputStream encryptionInputStream = new PrefixInputStream(source, packetLength, false); + byte[] packetIv = currentIvGenerator.newRandomUniqueIv(); + Cipher packetCipher = getPacketEncryptionCipher(packetIv); + encryptionInputStream = new CipherInputStream(encryptionInputStream, packetCipher); + encryptionInputStream = new SequenceInputStream(new ByteArrayInputStream(packetIv), encryptionInputStream); + encryptionInputStream = new BufferOnMarkInputStream(encryptionInputStream, packetLength); + return new CountingInputStream(encryptionInputStream, false); + } + + private Cipher getPacketEncryptionCipher(byte[] packetIv) throws IOException { + GCMParameterSpec gcmParameterSpec = new GCMParameterSpec(EncryptedRepository.GCM_TAG_SIZE_IN_BYTES * Byte.SIZE, packetIv); + try { + Cipher packetCipher = Cipher.getInstance(EncryptedRepository.GCM_ENCRYPTION_SCHEME); + packetCipher.init(Cipher.ENCRYPT_MODE, secretKey, gcmParameterSpec); + return packetCipher; + } catch (NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException | InvalidAlgorithmParameterException e) { + throw new IOException(e); + } + } + + public static long getEncryptionSize(long size, int packetLength) { + return size + (size / packetLength + 1) * (EncryptedRepository.GCM_TAG_SIZE_IN_BYTES + EncryptedRepository.GCM_IV_SIZE_IN_BYTES); + } + + static class IvGenerator { + + private final SecureRandom secureRandom; + private final Set previousIvs; + + IvGenerator() { + this.secureRandom = new SecureRandom(); + this.previousIvs = new HashSet<>(); + } + + IvGenerator(IvGenerator other) { + this(other.secureRandom, other.previousIvs); + } + + IvGenerator(SecureRandom secureRandom, Set previousIvs) { + try { + this.secureRandom = cloneRandom(secureRandom); + } catch (Exception e) { + throw new Error(e); + } + this.previousIvs = new HashSet<>(previousIvs); + } + + byte[] newRandomUniqueIv() { + byte[] iv = new byte[EncryptedRepository.GCM_TAG_SIZE_IN_BYTES]; + do { + secureRandom.nextBytes(iv); + } while (false == previousIvs.add(MessageDigests.toHexString(iv))); + return iv; + } + + private static SecureRandom cloneRandom(SecureRandom src) throws Exception { + ByteArrayOutputStream bo = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream(bo); + oos.writeObject(src); + oos.close(); + ObjectInputStream ois = new ObjectInputStream( + new ByteArrayInputStream(bo.toByteArray())); + return (SecureRandom)(ois.readObject()); + } + } + +} diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java new file mode 100644 index 0000000000000..0969c903c6f30 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStream.java @@ -0,0 +1,455 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.repositories.encrypted; + +import org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider; + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.SecretKey; +import javax.crypto.ShortBufferException; +import javax.crypto.spec.GCMParameterSpec; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.security.GeneralSecurityException; +import java.security.Provider; +import java.util.Objects; + +import static javax.crypto.Cipher.DECRYPT_MODE; +import static javax.crypto.Cipher.ENCRYPT_MODE; + +/** + * Given an {@code InputStream} and a {@code SecretKey}, creates a wrapper {@code InputStream} that encrypts/decrypts the content of the + * original stream. The original stream must not be otherwise used; it will be closed when the encryptor/decryptor stream is closed. + * + * The method of encryption is AES/GCM/NOPadding. The GCM mode is a form of authenticated encryption, meaning it offers authenticity in + * addition to the expected confidentiality. In other words, during decryption it verifies that the ciphertext being decrypted has not + * been tampered with. + * + * During encryption the source input stream is processed piece-wise and a packet (consisting of multiple pieces) of at most {@code + * GCMPacketsCipherInputStream#PACKET_SIZE_IN_BYTES} bytes size is encrypted and authenticated independently of the other packets. All + * packets in the same stream are encrypted with the same secret key, but a different IV, monotonically increasing with the packet index. + * Consequently, each packet has its own authentication tag appended, even the empty packet (all packets are the same size, but the last + * one can be of any size). The resulting ciphertext has a larger size than the source plaintext. + * + * Decryption also validates the authentication tag. It is important that the {@code Cipher} used during decryption, which is returned by + * the {@code Provider} parameter, NOT internally cache pieces of ciphertext, without releasing the decrypted plaintext, until it + * validates the associated authentication tag. Failure to comply to this requirement, will choke (throw {@code IllegalStateException}) the + * decryption stream, because the ciphertext is processed piece wise, the complete packet is not available fully at one moment. + * + * The resulting decrypted stream will return possibly un-authenticated content, but it is guaranteed that an {@code IOException} is + * thrown, at the latest when the stream has been exhausted ({@code InputStream#read} return {@code -1}) if the ciphertext has been altered. + * + * Both the encrypting and decrypting streams support {@code InputStream#mark} and {@code InputStream#reset}. Because GCM processing cannot + * be reset to a previous state, it only "goes forward", a mark call during the processing of a packet will buffer the processed bytes + * until the next packet boundary. A reset call will pick up any buffered data from a partially processed packet, and re-encrypt the + * following packets (the following packets of a mark call are not buffered, they are re-encrypted). + * + * This is obviously NOT thread-safe. + */ +public class GCMPacketsCipherInputStream extends FilterInputStream { + + static class GCMPacketsWithMarkCipherInputStream extends GCMPacketsCipherInputStream { + + private GCMPacketsWithMarkCipherInputStream(InputStream in, SecretKey secretKey, int mode, long packetIndex, int nonce, + Provider provider) { + super(in, secretKey, mode, packetIndex, nonce, provider); + } + + private ByteArrayOutputStream markWriteBuffer = null; + private ByteArrayInputStream markReadBuffer = new ByteArrayInputStream(new byte[0]); + private boolean markTriggeredForCurrentPacket = false; + private long markPacketIndex; + private int markReadLimit; + + @Override + public int read() throws IOException { + ensureOpen(); + // in case this is a reseted stream that has buffered part of the ciphertext + if (markReadBuffer.available() > 0) { + return markReadBuffer.read(); + } + int cipherByte = super.read(); + // if buffering of the ciphertext is required + if (markTriggeredForCurrentPacket && cipherByte != -1) { + markWriteBuffer.write(cipherByte); + } + return cipherByte; + } + + @Override + public int read(byte b[], int off, int len) throws IOException { + ensureOpen(); + // in case this is a reseted stream that has buffered part of the ciphertext + int bytesReadFromMarkBuffer = markReadBuffer.read(b, off, len); + if (bytesReadFromMarkBuffer != -1) { + return bytesReadFromMarkBuffer; + } + int cipherBytesCount = super.read(b, off, len); + // if buffering of the ciphertext is required + if (markTriggeredForCurrentPacket && cipherBytesCount > 0) { + markWriteBuffer.write(b, off, cipherBytesCount ); + } + return cipherBytesCount; + } + + @Override + public long skip(long n) throws IOException { + ensureOpen(); + if (markReadBuffer.available() > 0) { + return markReadBuffer.skip(n); + } + int bytesAvailable = super.available(); + bytesAvailable = Math.min(bytesAvailable, Math.toIntExact(n)); + if (markTriggeredForCurrentPacket) { + byte[] temp = new byte[bytesAvailable]; + int bytesRead = super.read(temp); + markWriteBuffer.write(temp); + return bytesRead; + } else { + return super.skip(n); + } + } + + @Override + public int available() throws IOException { + ensureOpen(); + return markReadBuffer.available() + super.available(); + } + + @Override + public boolean markSupported() { + return in.markSupported(); + } + + @Override + public void mark(int readLimit) { + markTriggeredForCurrentPacket = true; + markWriteBuffer = new ByteArrayOutputStream(); + if (markReadBuffer.available() > 0) { + markReadBuffer.mark(Integer.MAX_VALUE); + try { + markReadBuffer.transferTo(markWriteBuffer); + } catch (IOException e) { + throw new IllegalStateException(e); + } + markReadBuffer.reset(); + } + markPacketIndex = getPacketIndex(); + markReadLimit = readLimit; + } + + @Override + public void reset() throws IOException { + if (markWriteBuffer == null) { + throw new IOException("mark not called"); + } + if (markPacketIndex > getPacketIndex()) { + throw new IllegalStateException(); + } + // mark triggered before the packet boundary has been read over + if (false == markTriggeredForCurrentPacket) { + // reset underlying input stream to packet boundary + in.reset(); + // set packet index for the next packet and clear any transitory state of any inside of packet processing + setPacketIndex(markPacketIndex); + } + if (markPacketIndex != getPacketIndex()) { + throw new IllegalStateException(); + } + // make any cached ciphertext available to read + markReadBuffer = new ByteArrayInputStream(markWriteBuffer.toByteArray()); + } + + @Override + void readAtTheStartOfPacketHandler() { + if (markTriggeredForCurrentPacket) { + markTriggeredForCurrentPacket = false; + // mark the underlying stream at the start of the packet + in.mark(markReadLimit); + } + } + } + + private static final int GCM_TAG_SIZE_IN_BYTES = 16; + private static final int GCM_IV_SIZE_IN_BYTES = 12; + private static final String GCM_ENCRYPTION_SCHEME = "AES/GCM/NoPadding"; + + static final int PACKET_SIZE_IN_BYTES = 4096; + static final int ENCRYPTED_PACKET_SIZE_IN_BYTES = PACKET_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES; + static final int READ_BUFFER_SIZE_IN_BYTES = 512; + + private boolean done = false; + private boolean closed = false; + private final SecretKey secretKey; + private final int mode; + private final Provider provider; + + private Cipher packetCipher; + private long packetIndex; + private final ByteBuffer packetIV = ByteBuffer.allocate(GCM_IV_SIZE_IN_BYTES); + // how much to read from the underlying stream before finishing the current packet and starting the next one + private int stillToReadInPacket; + private final int packetSizeInBytes; + + private final byte[] inputByteBuffer = new byte[READ_BUFFER_SIZE_IN_BYTES]; + private final byte[] processedByteBuffer = new byte[READ_BUFFER_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES]; + private InputStream processedInputStream = InputStream.nullInputStream(); + private int bytesBufferedInsideTheCipher = 0; + + static GCMPacketsCipherInputStream getEncryptor(InputStream in, SecretKey secretKey, int nonce, Provider provider) { + return new GCMPacketsWithMarkCipherInputStream(in, secretKey, ENCRYPT_MODE, 0, nonce, provider); + } + + static GCMPacketsCipherInputStream getDecryptor(InputStream in, SecretKey secretKey, int nonce, Provider provider) { + return new GCMPacketsWithMarkCipherInputStream(in, secretKey, DECRYPT_MODE, 0, nonce, provider); + } + + public static GCMPacketsCipherInputStream getEncryptor(InputStream in, SecretKey secretKey, int nonce) { + return getEncryptor(in, secretKey, nonce, new BouncyCastleFipsProvider()); + } + + public static GCMPacketsCipherInputStream getDecryptor(InputStream in, SecretKey secretKey, int nonce) { + return getDecryptor(in, secretKey, nonce, new BouncyCastleFipsProvider()); + } + + public static long getEncryptionSizeFromPlainSize(long size) { + if (size < 0) { + throw new IllegalArgumentException(); + } + return (size / PACKET_SIZE_IN_BYTES) * (ENCRYPTED_PACKET_SIZE_IN_BYTES) + (size % PACKET_SIZE_IN_BYTES) + GCM_TAG_SIZE_IN_BYTES; + } + + public static long getDecryptionSizeFromCipherSize(long size) { + if (size < GCM_TAG_SIZE_IN_BYTES) { + throw new IllegalArgumentException(); + } + long plainSize = (size / (ENCRYPTED_PACKET_SIZE_IN_BYTES)) * PACKET_SIZE_IN_BYTES; + if (size % ENCRYPTED_PACKET_SIZE_IN_BYTES < GCM_TAG_SIZE_IN_BYTES) { + throw new IllegalArgumentException(); + } + return plainSize + (size % ENCRYPTED_PACKET_SIZE_IN_BYTES) - GCM_TAG_SIZE_IN_BYTES; + } + + private GCMPacketsCipherInputStream(InputStream in, SecretKey secretKey, int mode, long packetIndex, int nonce, Provider provider) { + super(Objects.requireNonNull(in)); + this.secretKey = Objects.requireNonNull(secretKey); + this.mode = mode; + this.packetIndex = packetIndex; + this.provider = provider; + // the first 8 bytes of the IV for packet encryption are the index of the packet + packetIV.putLong(packetIndex); + // the last 4 bytes of the IV for packet encryption are all equal (randomly generated) + packetIV.putInt(nonce); + if (mode == ENCRYPT_MODE) { + packetSizeInBytes = PACKET_SIZE_IN_BYTES; + } else { + packetSizeInBytes = PACKET_SIZE_IN_BYTES + GCM_TAG_SIZE_IN_BYTES; + } + stillToReadInPacket = packetSizeInBytes; + } + + private void reinitPacketCipher() throws GeneralSecurityException { + Cipher cipher; + if (provider != null) { + cipher = Cipher.getInstance(GCM_ENCRYPTION_SCHEME, provider); + } else { + cipher = Cipher.getInstance(GCM_ENCRYPTION_SCHEME); + } + // construct IV and increment packet index + packetIV.putLong(0, Math.incrementExact(packetIndex)); + GCMParameterSpec gcmParameterSpec = new GCMParameterSpec(GCM_TAG_SIZE_IN_BYTES * Byte.SIZE, packetIV.array()); + cipher.init(mode, secretKey, gcmParameterSpec); + packetCipher = cipher; + // the new cipher has no bytes buffered inside + bytesBufferedInsideTheCipher = 0; + } + + private int readAndProcess() throws IOException, GeneralSecurityException { + // do not read anything more, there are still processed bytes to be consumed + if (processedInputStream.available() > 0) { + processedInputStream.available(); + } + // the underlying input stream is exhausted + if (done) { + return -1; + } + // starting to read a new packet + if (stillToReadInPacket == packetSizeInBytes) { + // reinit cipher for this following packet + reinitPacketCipher(); + // call handler to notify subclasses that the processing of a new packet has started + readAtTheStartOfPacketHandler(); + } + int bytesToRead = Math.min(inputByteBuffer.length - bytesBufferedInsideTheCipher, stillToReadInPacket); + if (bytesToRead <= 0) { + throw new IllegalStateException(); + } + int bytesRead = in.read(inputByteBuffer, 0, bytesToRead); + assert bytesRead != 0 : "read must return at least one byte"; + assert processedInputStream.available() == 0 : "there exists processed still to be consumed, but it shouldn't"; + final int bytesProcessed; + if (bytesRead == -1) { + // end of the underlying stream to be encrypted + done = true; + try { + bytesProcessed = packetCipher.doFinal(processedByteBuffer, 0); + } catch (ShortBufferException e) { + throw new IllegalStateException(); + } + // there should be no internally buffered (by the cipher) data remaining after doFinal + bytesBufferedInsideTheCipher -= bytesProcessed; + if (mode == ENCRYPT_MODE) { + bytesBufferedInsideTheCipher += GCM_TAG_SIZE_IN_BYTES; + } else { + bytesBufferedInsideTheCipher -= GCM_TAG_SIZE_IN_BYTES; + } + if (bytesBufferedInsideTheCipher != 0) { + throw new IllegalStateException(); + } + } else { + stillToReadInPacket -= bytesRead; + if (stillToReadInPacket < 0) { + throw new IllegalStateException(); + } + if (stillToReadInPacket == 0) { + // this is the last encryption for this packet + try { + bytesProcessed = packetCipher.doFinal(inputByteBuffer, 0, bytesRead, processedByteBuffer, 0); + } catch (ShortBufferException e) { + throw new IllegalStateException(e); + } + // there should be no internally buffered (by the cipher) data remaining after doFinal + bytesBufferedInsideTheCipher += (bytesRead - bytesProcessed); + if (mode == ENCRYPT_MODE) { + bytesBufferedInsideTheCipher += GCM_TAG_SIZE_IN_BYTES; + } else { + bytesBufferedInsideTheCipher -= GCM_TAG_SIZE_IN_BYTES; + } + if (bytesBufferedInsideTheCipher != 0) { + throw new IllegalArgumentException(); + } + // reset the packet size for the next packet + stillToReadInPacket = packetSizeInBytes; + } else { + // this is a partial encryption inside the packet + try { + bytesProcessed = packetCipher.update(inputByteBuffer, 0, bytesRead, processedByteBuffer, 0); + } catch (ShortBufferException e) { + throw new IllegalStateException(e); + } + // the cipher might encrypt only part of the plaintext and cache the rest + bytesBufferedInsideTheCipher += (bytesRead - bytesProcessed); + } + } + // the "if" here is just an "optimization" + if (bytesProcessed != 0) { + processedInputStream = new ByteArrayInputStream(processedByteBuffer, 0, bytesProcessed); + } + return bytesProcessed; + } + + @Override + public int read() throws IOException { + while (processedInputStream.available() <= 0) { + try { + if (readAndProcess() == -1) { + return -1; + } + } catch (GeneralSecurityException e) { + throw new IOException(e); + } + } + return processedInputStream.read(); + } + + @Override + public int read(byte b[], int off, int len) throws IOException { + while (processedInputStream.available() <= 0) { + try { + if (readAndProcess() == -1) { + return -1; + } + } catch (GeneralSecurityException e) { + throw new IOException(e); + } + } + return processedInputStream.read(b, off, len); + } + + @Override + public long skip(long n) throws IOException { + return processedInputStream.skip(n); + } + + @Override + public int available() throws IOException { + return processedInputStream.available(); + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + closed = true; + processedInputStream = InputStream.nullInputStream(); + in.close(); + // Throw away the unprocessed data and throw no crypto exceptions. + // Normally the GCM cipher is fully readed before closing, so any authentication + // exceptions would occur while reading. + if (false == done) { + done = true; + try { + packetCipher.doFinal(); + } catch (BadPaddingException | IllegalBlockSizeException ex) { + // Catch exceptions as the rest of the stream is unused. + } + } + } + + @Override + public boolean markSupported() { + return false; + } + + @Override + public void mark(int readLimit) { + } + + @Override + public void reset() throws IOException { + throw new IOException("mark/reset not supported"); + } + + /** + * Sets the packet index and clears the transitory state from processing of the previous packet + */ + void setPacketIndex(long packetIndex) { + processedInputStream = InputStream.nullInputStream(); + stillToReadInPacket = packetSizeInBytes; + done = false; + this.packetIndex = packetIndex; + } + + long getPacketIndex() { + return packetIndex; + } + + void readAtTheStartOfPacketHandler() { + } + + void ensureOpen() throws IOException { + if (closed) { + throw new IOException("Stream closed"); + } + } +} diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsDecryptorInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsDecryptorInputStream.java new file mode 100644 index 0000000000000..884622d0ab80f --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsDecryptorInputStream.java @@ -0,0 +1,172 @@ +package org.elasticsearch.repositories.encrypted; + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; +import javax.crypto.ShortBufferException; +import javax.crypto.spec.GCMParameterSpec; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.util.Iterator; + +public class GCMPacketsDecryptorInputStream extends FilterInputStream { + + private final int maxPacketSizeInBytes; + private final int authenticationTagSizeInBytes; + private final SecretKey secretKey; + private final Iterator packetInfoIterator; + private final byte[] packetBuffer; + + private int packetIndex; + private int bufferStartOffset; + private int bufferEndOffset; + private boolean closed; + + protected GCMPacketsDecryptorInputStream(InputStream in, SecretKey secretKey, int maxPacketSizeInBytes, + int authenticationTagSizeInBytes, + Iterator packetInfoIterator) { + super(in); + this.secretKey = secretKey; + this.maxPacketSizeInBytes = maxPacketSizeInBytes; + this.authenticationTagSizeInBytes = authenticationTagSizeInBytes; + this.packetInfoIterator = packetInfoIterator; + this.packetBuffer = new byte[maxPacketSizeInBytes + authenticationTagSizeInBytes]; + this.bufferStartOffset = 0; + this.bufferEndOffset = 0; + this.closed = false; + } + + @Override + public int read() throws IOException { + if (bufferStartOffset >= bufferEndOffset) { + bufferEndOffset = readAndDecryptNextPacket(); + if (bufferEndOffset == -1) { + return -1; + } + bufferStartOffset = 0; + } + return packetBuffer[bufferStartOffset++]; + } + + @Override + public int read(byte b[], int off, int len) throws IOException { + if (off < 0 || len < 0 || len > b.length - off) { + throw new IndexOutOfBoundsException(); + } + if (len == 0) { + return 0; + } + if (bufferStartOffset >= bufferEndOffset) { + bufferEndOffset = readAndDecryptNextPacket(); + if (bufferEndOffset == -1) { + return -1; + } + bufferStartOffset = 0; + } + int readSize = Math.min(len, bufferEndOffset - bufferStartOffset); + System.arraycopy(packetBuffer, bufferStartOffset, b, off, readSize); + bufferStartOffset += readSize; + return readSize; + } + + @Override + public long skip(long n) throws IOException { + if (n == 0L) { + return 0L; + } + if (bufferStartOffset >= bufferEndOffset) { + bufferEndOffset = readAndDecryptNextPacket(); + if (bufferEndOffset == -1) { + return 0; + } + bufferStartOffset = 0; + } + int skipSize = Math.toIntExact(Math.min(n, bufferEndOffset - bufferStartOffset)); + bufferStartOffset += skipSize; + return skipSize; + } + + @Override + public int available() throws IOException { + return Math.max(0, bufferEndOffset - bufferStartOffset); + } + + @Override + public boolean markSupported() { + return false; + } + + @Override + public void mark(int readLimit) { + } + + @Override + public void reset() throws IOException { + throw new IOException("mark/reset not supported"); + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + closed = true; + in.close(); + } + + void ensureOpen() throws IOException { + if (closed) { + throw new IOException("Stream closed"); + } + } + + private int readAndDecryptNextPacket() throws IOException { + if (false == packetInfoIterator.hasNext()) { + return -1; + } + BlobEncryptionMetadata.PacketInfo currentPacketInfo = packetInfoIterator.next(); + int packetSize = currentPacketInfo.getSizeInBytes(); + if (packetSize > maxPacketSizeInBytes) { + throw new IllegalArgumentException(); + } + ensureOpen(); + int bytesRead = in.readNBytes(packetBuffer, 0, packetSize); + if (bytesRead != packetSize) { + throw new IllegalArgumentException(); + } + if (currentPacketInfo.getAuthTag().length != authenticationTagSizeInBytes) { + throw new IllegalArgumentException(); + } + System.arraycopy(currentPacketInfo.getAuthTag(), 0, packetBuffer, packetSize, authenticationTagSizeInBytes); + Cipher packetCipher = getPacketDecryptionCipher(currentPacketInfo.getIv()); + final int bytesDecrypted; + try { + // in-place decryption + bytesDecrypted = packetCipher.doFinal(packetBuffer, 0, packetSize + authenticationTagSizeInBytes, packetBuffer); + } catch (ShortBufferException | IllegalBlockSizeException | BadPaddingException e) { + throw new IOException(e); + } + if (bytesDecrypted != packetSize) { + throw new IllegalStateException(); + } + return packetSize; + } + + private Cipher getPacketDecryptionCipher(byte[] packetIv) throws IOException { + GCMParameterSpec gcmParameterSpec = new GCMParameterSpec(EncryptedRepository.GCM_TAG_SIZE_IN_BYTES * Byte.SIZE, packetIv); + try { + Cipher packetCipher = Cipher.getInstance(EncryptedRepository.GCM_ENCRYPTION_SCHEME); + packetCipher.init(Cipher.DECRYPT_MODE, secretKey, gcmParameterSpec); + return packetCipher; + } catch (NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException | InvalidAlgorithmParameterException e) { + throw new IOException(e); + } + } + +} diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java new file mode 100644 index 0000000000000..abf3a56b8f3fb --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/GCMPacketsEncryptorInputStream.java @@ -0,0 +1,276 @@ +package org.elasticsearch.repositories.encrypted; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; +import javax.crypto.ShortBufferException; +import javax.crypto.spec.GCMParameterSpec; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +// not thread-safe +public class GCMPacketsEncryptorInputStream extends FilterInputStream { + + private final Logger logger = LogManager.getLogger(getClass()); + private final int maxPacketSizeInBytes; + private final byte[] packetTrailByteBuffer; + private final SecretKey secretKey; + private final IvRandomUniqueGenerator ivGenerator; + private final List packetInfoList; + + private int bytesRemainingInPacket; + private byte[] packetIv; + private Cipher packetCipher; + private boolean closed; + private int markPacketIndex; + + protected GCMPacketsEncryptorInputStream(InputStream in, SecretKey secretKey, int maxPacketSizeInBytes) throws IOException { + super(in); + this.maxPacketSizeInBytes = maxPacketSizeInBytes; + this.packetTrailByteBuffer = new byte[EncryptedRepository.AES_BLOCK_SIZE_IN_BYTES + EncryptedRepository.GCM_TAG_SIZE_IN_BYTES]; + this.secretKey = secretKey; + this.ivGenerator = new IvRandomUniqueGenerator(); + this.packetInfoList = new ArrayList<>(); + this.bytesRemainingInPacket = maxPacketSizeInBytes; + this.packetIv = ivGenerator.newUniqueIv(); + this.packetCipher = getPacketEncryptionCipher(packetIv); + this.closed = false; + this.markPacketIndex = -1; + } + + @Override + public int read(byte b[], int off, int len) throws IOException { + ensureOpen(); + int maxReadSize = getReadSize(len); + int readSize = in.readNBytes(b, off, maxReadSize); + assert readSize >= 0 : "readNBytes does not return -1 on end-of-stream"; + if (readSize == 0) { + if (maxReadSize == 0) { + // 0 bytes were requested + return 0; + } + // end of filtered input stream + assert maxReadSize > 0; + assert in.read() == -1 : "readNBytes returned no bytes but it's not the end-of-stream"; + return -1; + } + bytesRemainingInPacket -= readSize; + final int encryptedSize; + try { + // in-place encryption + encryptedSize = packetCipher.update(b, off, readSize, b, off); + } catch (ShortBufferException e) { + throw new IllegalStateException(e); + } + if (bytesRemainingInPacket == 0 || readSize % EncryptedRepository.AES_BLOCK_SIZE_IN_BYTES != 0) { + // finalize packet + final byte[] authenticationTag; + if (encryptedSize == readSize) { + try { + authenticationTag = packetCipher.doFinal(); + } catch (IllegalBlockSizeException | BadPaddingException e) { + throw new IOException(e); + } + } else { + if (readSize - encryptedSize >= EncryptedRepository.AES_BLOCK_SIZE_IN_BYTES) { + throw new IllegalStateException(); + } + int trailAndTagSize = 0; + try { + trailAndTagSize = packetCipher.doFinal(packetTrailByteBuffer, 0); + } catch (IllegalBlockSizeException | ShortBufferException | BadPaddingException e) { + throw new IOException(e); + } + if (encryptedSize + trailAndTagSize != readSize + EncryptedRepository.GCM_TAG_SIZE_IN_BYTES) { + throw new IllegalStateException(); + } + // copy the remaining packet trail bytes + System.arraycopy(packetTrailByteBuffer, 0, b, off + encryptedSize, trailAndTagSize - EncryptedRepository.GCM_TAG_SIZE_IN_BYTES); + authenticationTag = Arrays.copyOfRange(packetTrailByteBuffer, trailAndTagSize - EncryptedRepository.GCM_TAG_SIZE_IN_BYTES, trailAndTagSize); + } + if (authenticationTag.length != EncryptedRepository.GCM_TAG_SIZE_IN_BYTES) { + throw new IllegalStateException(); + } + finishPacket(authenticationTag); + return readSize; + } else { + if (encryptedSize != readSize) { + throw new IllegalStateException(); + } + return readSize; + } + } + + @Override + public int read() throws IOException { + byte[] b = new byte[1]; + int bytesRead = read(b, 0, 1); + if (bytesRead == -1) { + return -1; + } + if (bytesRead != 1) { + throw new IllegalStateException(); + } + return (int) b[0]; + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + closed = true; + in.close(); + if (bytesRemainingInPacket < maxPacketSizeInBytes) { + // finish last packet + final byte[] authenticationTag; + try { + authenticationTag = packetCipher.doFinal(); + } catch (IllegalBlockSizeException | BadPaddingException e) { + throw new IOException(e); + } + finishPacket(authenticationTag); + } + } + + @Override + public void mark(int readLimit) { + in.mark(readLimit); + if (bytesRemainingInPacket < maxPacketSizeInBytes) { + // finish in-progress packet + try { + byte[] authenticationTag = packetCipher.doFinal(); + finishPacket(authenticationTag); + } catch (IllegalBlockSizeException | BadPaddingException e) { + throw new UncheckedIOException(new IOException(e)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + markPacketIndex = packetInfoList.size(); + } + + @Override + public void reset() throws IOException { + ensureOpen(); + in.reset(); + // discard packets after mark point + packetInfoList.subList(markPacketIndex, packetInfoList.size()).clear(); + // reinstantiate packetCipher + bytesRemainingInPacket = maxPacketSizeInBytes; + packetIv = ivGenerator.newUniqueIv(); + packetCipher = getPacketEncryptionCipher(packetIv); + } + + public List getEncryptionPacketMetadata() { + if (false == closed) { + throw new IllegalStateException("Stream must be closed in order to completely assemble the metadata"); + } + return Collections.unmodifiableList(packetInfoList); + } + + void ensureOpen() throws IOException { + if (closed) { + throw new IOException("Stream closed"); + } + } + + private void finishPacket(byte[] authTag) throws IOException { + packetInfoList.add(new BlobEncryptionMetadata.PacketInfo(packetIv, authTag, maxPacketSizeInBytes - bytesRemainingInPacket)); + bytesRemainingInPacket = maxPacketSizeInBytes; + packetIv = ivGenerator.newUniqueIv(); + packetCipher = getPacketEncryptionCipher(packetIv); + } + + private Cipher getPacketEncryptionCipher(byte[] packetIv) throws IOException { + GCMParameterSpec gcmParameterSpec = new GCMParameterSpec(EncryptedRepository.GCM_TAG_SIZE_IN_BYTES * Byte.SIZE, packetIv); + try { + Cipher packetCipher = Cipher.getInstance(EncryptedRepository.GCM_ENCRYPTION_SCHEME); + packetCipher.init(Cipher.ENCRYPT_MODE, secretKey, gcmParameterSpec); + return packetCipher; + } catch (NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException | InvalidAlgorithmParameterException e) { + throw new IOException(e); + } + } + + /** + * Tries to return a read size value such that it is smaller or equal to the requested {@code len}, does not exceed the remaining + * space in the current packet and, very important, is a multiple of {@link EncryptedRepository#AES_BLOCK_SIZE_IN_BYTES}. If the + * requested {@code len} or the remaining space in the current packet are smaller than + * {@link EncryptedRepository#AES_BLOCK_SIZE_IN_BYTES}, then their minimum is returned. + * + * @param len the requested read size + */ + private int getReadSize(int len) { + if (bytesRemainingInPacket <= 0) { + throw new IllegalStateException(); + } + if (len < 0) { + throw new IndexOutOfBoundsException(); + } + if (len == 0) { + return 0; + } + int maxReadSize = Math.min(len, bytesRemainingInPacket); + int readSize = (maxReadSize / EncryptedRepository.AES_BLOCK_SIZE_IN_BYTES) * EncryptedRepository.AES_BLOCK_SIZE_IN_BYTES; + if (readSize != 0) { + return readSize; + } + assert maxReadSize < EncryptedRepository.AES_BLOCK_SIZE_IN_BYTES; + if (maxReadSize == len) { + logger.warn("Reading [" + len + "] bytes, which is less than [" + EncryptedRepository.AES_BLOCK_SIZE_IN_BYTES + "], is terribly inefficient."); + } + return maxReadSize; + } + + static class IvRandomUniqueGenerator { + + private final Map> generatedIvs; + private final SecureRandom secureRandom; + + IvRandomUniqueGenerator() { + generatedIvs = new HashMap<>(); + secureRandom = new SecureRandom(); + } + + byte[] newUniqueIv() { + return newUniqueIv(5); + } + + private byte[] newUniqueIv(int retryCount) { + if (retryCount <= 0) { + throw new IllegalStateException("Secure random returns many similar values"); + } + long part1 = secureRandom.nextLong(); + Set part2Set = generatedIvs.computeIfAbsent(part1, k -> new HashSet<>()); + int part2 = secureRandom.nextInt(); + if (false == part2Set.add(part2)) { + return newUniqueIv(retryCount - 1); + } + ByteBuffer uniqueIv = ByteBuffer.allocate(EncryptedRepository.GCM_IV_SIZE_IN_BYTES); + uniqueIv.putLong(part1); + uniqueIv.putInt(part2); + return uniqueIv.array(); + } + } +} diff --git a/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/PrefixInputStream.java b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/PrefixInputStream.java new file mode 100644 index 0000000000000..8cf280126d915 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/PrefixInputStream.java @@ -0,0 +1,106 @@ +package org.elasticsearch.repositories.encrypted; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Objects; + +public final class PrefixInputStream extends FilterInputStream { + + private final int length; + private int position; + private boolean closeSource; + private boolean closed; + + public PrefixInputStream(InputStream in, int length, boolean closeSource) { + super(Objects.requireNonNull(in)); + this.length = length; + this.position = 0; + this.closeSource = closeSource; + this.closed = false; + } + + @Override + public int read() throws IOException { + ensureOpen(); + if (position >= length) { + return -1; + } + int byteVal = in.read(); + if (byteVal == -1) { + return -1; + } + position++; + return byteVal; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + ensureOpen(); + Objects.checkFromIndexSize(off, len, b.length); + if (len == 0) { + return 0; + } + if (position >= length) { + return -1; + } + int readSize = Math.min(len, length - position); + int bytesRead = in.read(b, off, readSize); + if (bytesRead == -1) { + return -1; + } + position += bytesRead; + return bytesRead; + } + + @Override + public long skip(long n) throws IOException { + ensureOpen(); + if (n <= 0 || position >= length) { + return 0; + } + long bytesToSkip = Math.min(n, length - position); + assert bytesToSkip > 0; + long bytesSkipped = in.skip(bytesToSkip); + position += bytesSkipped; + return bytesSkipped; + } + + @Override + public int available() throws IOException { + ensureOpen(); + return Math.min(length - position, super.available()); + } + + @Override + public void mark(int readlimit) { + } + + @Override + public void reset() throws IOException { + throw new IOException("mark/reset not supported"); + } + + @Override + public boolean markSupported() { + return false; + } + + private void ensureOpen() throws IOException { + if (closed) { + throw new IOException("Stream has been closed"); + } + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + closed = true; + if (closeSource) { + in.close(); + } + } + +} diff --git a/x-pack/plugin/repository-encrypted/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/repository-encrypted/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 0000000000000..9aa4d16b19a80 --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +grant { + permission java.security.SecurityPermission "putProviderProperty.BCFIPS"; + permission java.lang.RuntimePermission "accessDeclaredMembers"; + permission java.lang.RuntimePermission "accessClassInPackage.sun.security.provider"; + permission org.bouncycastle.crypto.CryptoServicesPermission "exportSecretKey"; +}; diff --git a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/EncryptedRepositoryTests.java b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/EncryptedRepositoryTests.java new file mode 100644 index 0000000000000..e65120d749fcd --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/EncryptedRepositoryTests.java @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.repositories.encrypted; + +import org.elasticsearch.test.ESTestCase; + +public class EncryptedRepositoryTests extends ESTestCase { + public void testThatDoesNothing() { + } +} diff --git a/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java new file mode 100644 index 0000000000000..c90f5f3d821ce --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/test/java/org/elasticsearch/repositories/encrypted/GCMPacketsCipherInputStreamTests.java @@ -0,0 +1,389 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.repositories.encrypted; + +import org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider; +import org.elasticsearch.common.Randomness; +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; +import org.junit.Before; +import org.junit.BeforeClass; + +import javax.crypto.AEADBadTagException; +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.security.SecureRandom; +import java.util.Arrays; +import java.util.Random; + +import static org.elasticsearch.repositories.encrypted.GCMPacketsCipherInputStream.ENCRYPTED_PACKET_SIZE_IN_BYTES; +import static org.elasticsearch.repositories.encrypted.GCMPacketsCipherInputStream.READ_BUFFER_SIZE_IN_BYTES; + +public class GCMPacketsCipherInputStreamTests extends ESTestCase { + + private static int TEST_ARRAY_SIZE = 8 * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES; + private static byte[] testPlaintextArray; + private static BouncyCastleFipsProvider bcFipsProvider; + private SecretKey secretKey; + + @BeforeClass + static void setupProvider() { + AccessController.doPrivileged((PrivilegedAction) () -> { + GCMPacketsCipherInputStreamTests.bcFipsProvider = new BouncyCastleFipsProvider(); + return null; + }); + testPlaintextArray = new byte[TEST_ARRAY_SIZE]; + Randomness.get().nextBytes(testPlaintextArray); + } + + @Before + void createSecretKey() throws Exception { + secretKey = generateSecretKey(); + } + + public void testEncryptDecryptEmpty() throws Exception { + testEncryptDecryptRandomOfLength(0, secretKey); + } + + public void testEncryptDecryptSmallerThanBufferSize() throws Exception { + for (int i = 1; i < GCMPacketsCipherInputStream.READ_BUFFER_SIZE_IN_BYTES; i++) { + testEncryptDecryptRandomOfLength(i, secretKey); + } + } + + public void testEncryptDecryptMultipleOfBufferSize() throws Exception { + for (int i = 1; i < 10; i++) { + testEncryptDecryptRandomOfLength(i * GCMPacketsCipherInputStream.READ_BUFFER_SIZE_IN_BYTES, secretKey); + } + } + + public void testEncryptDecryptSmallerThanPacketSize() throws Exception { + for (int i = GCMPacketsCipherInputStream.READ_BUFFER_SIZE_IN_BYTES + 1; i < GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES; i++) { + testEncryptDecryptRandomOfLength(i, secretKey); + } + } + + public void testEncryptDecryptLargerThanPacketSize() throws Exception { + for (int i = GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES + 1; i <= GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES * 3; i++) { + testEncryptDecryptRandomOfLength(i, secretKey); + } + } + + public void testEncryptDecryptMultipleOfPacketSize() throws Exception { + for (int i = 1; i <= 6; i++) { + testEncryptDecryptRandomOfLength(i * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES, secretKey); + } + } + + public void testMarkAndResetAtBeginningForEncryption() throws Exception { + testMarkAndResetToSameOffsetForEncryption(0); + testMarkAndResetToSameOffsetForEncryption(Math.toIntExact(GCMPacketsCipherInputStream. + getEncryptionSizeFromPlainSize(GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES))); + } + + public void testMarkAndResetFirstPacketForEncryption() throws Exception { + for (int i = 1; i < ENCRYPTED_PACKET_SIZE_IN_BYTES; i++) { + testMarkAndResetToSameOffsetForEncryption(i); + } + } + + public void testMarkAndResetRandomSecondPacketForEncryption() throws Exception { + for (int i = ENCRYPTED_PACKET_SIZE_IN_BYTES + 1; i < 2 * ENCRYPTED_PACKET_SIZE_IN_BYTES; i++) { + testMarkAndResetToSameOffsetForEncryption(i); + } + } + + public void testMarkAndResetCrawlForEncryption() throws Exception { + int length = GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES; + int startIndex = randomIntBetween(0, testPlaintextArray.length - length); + int nonce = Randomness.get().nextInt(); + byte[] ciphertextBytes; + try (InputStream cipherInputStream = + GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), + secretKey, nonce, bcFipsProvider)) { + ciphertextBytes = cipherInputStream.readAllBytes(); + } + try (InputStream cipherInputStream = + GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), + secretKey, nonce, bcFipsProvider)) { + cipherInputStream.mark(Integer.MAX_VALUE); + for (int i = 0; i < GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length); i++) { + int skipSize = randomIntBetween(1, Math.toIntExact(GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length)) - i); + // skip bytes + cipherInputStream.readNBytes(skipSize); + cipherInputStream.reset(); + // re-read one byte of the skipped bytes + int byteRead = cipherInputStream.read(); + // mark the one byte progress + cipherInputStream.mark(Integer.MAX_VALUE); + assertThat("Mismatch at position: " + i, (byte) byteRead, Matchers.is(ciphertextBytes[i])); + } + } + } + + public void testMarkAndResetStepInRewindBuffer() throws Exception { + int length = 2 * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES; + int startIndex = randomIntBetween(0, testPlaintextArray.length - length); + int nonce = Randomness.get().nextInt(); + byte[] ciphertextBytes; + try (InputStream cipherInputStream = + GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), + secretKey, nonce, bcFipsProvider)) { + ciphertextBytes = cipherInputStream.readAllBytes(); + } + try (InputStream cipherInputStream = + GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), + secretKey, nonce, bcFipsProvider)) { + int position1 = randomIntBetween(1, ENCRYPTED_PACKET_SIZE_IN_BYTES - 2); + int position2 = randomIntBetween(position1 + 1, ENCRYPTED_PACKET_SIZE_IN_BYTES - 1); + int position3 = ENCRYPTED_PACKET_SIZE_IN_BYTES; + int position4 = randomIntBetween(position3 + 1, 2 * ENCRYPTED_PACKET_SIZE_IN_BYTES - 2); + int position5 = randomIntBetween(position4 + 1, 2 * ENCRYPTED_PACKET_SIZE_IN_BYTES - 1); + int position6 = 2 * ENCRYPTED_PACKET_SIZE_IN_BYTES; + int position7 = Math.toIntExact(GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length)); + // skip position1 bytes + cipherInputStream.readNBytes(position1); + // mark position1 + cipherInputStream.mark(Integer.MAX_VALUE); + byte[] bytesPos17; + if (randomBoolean()) { + bytesPos17 = cipherInputStream.readAllBytes(); + } else { + bytesPos17 = cipherInputStream.readNBytes(position7 - position1); + } + // reset back to position 1 + cipherInputStream.reset(); + byte[] bytesPos12 = cipherInputStream.readNBytes(position2 - position1); + assertTrue(Arrays.equals(bytesPos12, 0, bytesPos12.length, bytesPos17, 0, bytesPos12.length)); + // mark position2 + cipherInputStream.mark(Integer.MAX_VALUE); + byte[] bytesPos26 = cipherInputStream.readNBytes(position6 - position2); + assertTrue(Arrays.equals(bytesPos26, 0, bytesPos26.length, bytesPos17, (position2 - position1), + (position2 - position1) + bytesPos26.length)); + // reset to position 2 + cipherInputStream.reset(); + byte[] bytesPos23 = cipherInputStream.readNBytes(position3 - position2); + assertTrue(Arrays.equals(bytesPos23, 0, bytesPos23.length, bytesPos17, (position2 - position1), + (position2 - position1) + bytesPos23.length)); + // mark position3 + cipherInputStream.mark(Integer.MAX_VALUE); + byte[] bytesPos36 = cipherInputStream.readNBytes(position6 - position3); + assertTrue(Arrays.equals(bytesPos36, 0, bytesPos36.length, bytesPos17, (position3 - position1), + (position3 - position1) + bytesPos36.length)); + // reset to position 3 + cipherInputStream.reset(); + byte[] bytesPos34 = cipherInputStream.readNBytes(position4 - position3); + assertTrue(Arrays.equals(bytesPos34, 0, bytesPos34.length, bytesPos17, (position3 - position1), + (position3 - position1) + bytesPos34.length)); + // mark position4 + cipherInputStream.mark(Integer.MAX_VALUE); + byte[] bytesPos46 = cipherInputStream.readNBytes(position6 - position4); + assertTrue(Arrays.equals(bytesPos46, 0, bytesPos46.length, bytesPos17, (position4 - position1), + (position4 - position1) + bytesPos46.length)); + // reset to position 4 + cipherInputStream.reset(); + byte[] bytesPos45 = cipherInputStream.readNBytes(position5 - position4); + assertTrue(Arrays.equals(bytesPos45, 0, bytesPos45.length, bytesPos17, (position4 - position1), + (position4 - position1) + bytesPos45.length)); + // mark position 5 + cipherInputStream.mark(Integer.MAX_VALUE); + byte[] bytesPos56 = cipherInputStream.readNBytes(position6 - position5); + assertTrue(Arrays.equals(bytesPos56, 0, bytesPos56.length, bytesPos17, (position5 - position1), + (position5 - position1) + bytesPos56.length)); + // mark position 6 + cipherInputStream.mark(Integer.MAX_VALUE); + byte[] bytesPos67; + if (randomBoolean()) { + bytesPos67 = cipherInputStream.readAllBytes(); + } else { + bytesPos67 = cipherInputStream.readNBytes(position7 - position6); + } + assertTrue(Arrays.equals(bytesPos67, 0, bytesPos67.length, bytesPos17, (position6 - position1), + (position6 - position1) + bytesPos67.length)); + // mark position 7 (end of stream) + cipherInputStream.mark(Integer.MAX_VALUE); + // end of stream + assertThat(cipherInputStream.read(), Matchers.is(-1)); + // reset at the end + cipherInputStream.reset(); + assertThat(cipherInputStream.read(), Matchers.is(-1)); + } + } + + public void testDecryptionFails() throws Exception { + Random random = Randomness.get(); + int length = randomIntBetween(0, READ_BUFFER_SIZE_IN_BYTES); + int startIndex = randomIntBetween(0, testPlaintextArray.length - length); + int nonce = random.nextInt(); + byte[] ciphertextBytes; + try (InputStream cipherInputStream = + GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), + secretKey, nonce, bcFipsProvider)) { + ciphertextBytes = cipherInputStream.readAllBytes(); + } + // decryption fails for one byte modifications + for (int i = 0; i < ciphertextBytes.length; i++) { + byte bytei = ciphertextBytes[i]; + while (bytei == ciphertextBytes[i]) { + ciphertextBytes[i] = (byte) random.nextInt(); + } + try (InputStream plainInputStream = + GCMPacketsCipherInputStream.getDecryptor(new ByteArrayInputStream(ciphertextBytes), + secretKey, nonce, bcFipsProvider)) { + IOException e = expectThrows(IOException.class, () -> { + readAllInputStream(plainInputStream, + GCMPacketsCipherInputStream.getDecryptionSizeFromCipherSize(ciphertextBytes.length)); + }); + assertThat(e.getCause(), Matchers.isA(AEADBadTagException.class)); + } + ciphertextBytes[i] = bytei; + } + // decryption fails for one byte omissions + byte[] missingByteCiphertext = new byte[ciphertextBytes.length - 1]; + for (int i = 0; i < ciphertextBytes.length; i++) { + System.arraycopy(ciphertextBytes, 0, missingByteCiphertext, 0, i); + System.arraycopy(ciphertextBytes, i + 1, missingByteCiphertext, i, (ciphertextBytes.length - i - 1)); + try (InputStream plainInputStream = + GCMPacketsCipherInputStream.getDecryptor(new ByteArrayInputStream(missingByteCiphertext), + secretKey, nonce, bcFipsProvider)) { + IOException e = expectThrows(IOException.class, () -> { + readAllInputStream(plainInputStream, + GCMPacketsCipherInputStream.getDecryptionSizeFromCipherSize(missingByteCiphertext.length)); + }); + assertThat(e.getCause(), Matchers.isA(AEADBadTagException.class)); + } + } + // decryption fails for one extra byte + byte[] extraByteCiphertext = new byte[ciphertextBytes.length + 1]; + for (int i = 0; i < ciphertextBytes.length; i++) { + System.arraycopy(ciphertextBytes, 0, extraByteCiphertext, 0, i); + extraByteCiphertext[i] = (byte) random.nextInt(); + System.arraycopy(ciphertextBytes, i, extraByteCiphertext, i + 1, (ciphertextBytes.length - i)); + try (InputStream plainInputStream = + GCMPacketsCipherInputStream.getDecryptor(new ByteArrayInputStream(extraByteCiphertext), + secretKey, nonce, bcFipsProvider)) { + IOException e = expectThrows(IOException.class, () -> { + readAllInputStream(plainInputStream, + GCMPacketsCipherInputStream.getDecryptionSizeFromCipherSize(extraByteCiphertext.length)); + }); + assertThat(e.getCause(), Matchers.isA(AEADBadTagException.class)); + } + } + } + + private void testMarkAndResetToSameOffsetForEncryption(int offset) throws Exception { + int length = 4 * GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES + randomIntBetween(0, + GCMPacketsCipherInputStream.PACKET_SIZE_IN_BYTES); + int startIndex = randomIntBetween(0, testPlaintextArray.length - length); + try (InputStream cipherInputStream = + GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), + secretKey, Randomness.get().nextInt(), bcFipsProvider)) { + // skip offset bytes + cipherInputStream.readNBytes(offset); + // mark after offset + cipherInputStream.mark(Integer.MAX_VALUE); + // read/skip less than (encrypted) packet size + int skipSize = randomIntBetween(1, ENCRYPTED_PACKET_SIZE_IN_BYTES - 1); + byte[] firstPassEncryption = cipherInputStream.readNBytes(skipSize); + // back to start + cipherInputStream.reset(); + // read/skip more than (encrypted) packet size, but less than the full stream + skipSize = randomIntBetween(ENCRYPTED_PACKET_SIZE_IN_BYTES, + Math.toIntExact(GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize((long)length)) - 1 - offset); + byte[] secondPassEncryption = cipherInputStream.readNBytes(skipSize); + assertTrue(Arrays.equals(firstPassEncryption, 0, firstPassEncryption.length, secondPassEncryption, 0, + firstPassEncryption.length)); + // back to start + cipherInputStream.reset(); + byte[] thirdPassEncryption; + // read/skip to end of ciphertext + if (randomBoolean()) { + thirdPassEncryption = cipherInputStream.readAllBytes(); + } else { + thirdPassEncryption = cipherInputStream.readNBytes( + Math.toIntExact(GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize((long) length)) - offset); + } + assertTrue(Arrays.equals(secondPassEncryption, 0, secondPassEncryption.length, thirdPassEncryption, 0, + secondPassEncryption.length)); + // back to start + cipherInputStream.reset(); + // read/skip more than (encrypted) packet size, but less than the full stream + skipSize = randomIntBetween(ENCRYPTED_PACKET_SIZE_IN_BYTES, + Math.toIntExact(GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize((long)length)) - 1 - offset); + byte[] fourthPassEncryption = cipherInputStream.readNBytes(skipSize); + assertTrue(Arrays.equals(fourthPassEncryption, 0, fourthPassEncryption.length, thirdPassEncryption, 0, + fourthPassEncryption.length)); + // back to start + cipherInputStream.reset(); + // read/skip less than (encrypted) packet size + skipSize = randomIntBetween(1, ENCRYPTED_PACKET_SIZE_IN_BYTES - 1); + byte[] fifthsPassEncryption = cipherInputStream.readNBytes(skipSize); + assertTrue(Arrays.equals(fifthsPassEncryption, 0, fifthsPassEncryption.length, fourthPassEncryption, 0, + fifthsPassEncryption.length)); + } + } + + private void testEncryptDecryptRandomOfLength(int length, SecretKey secretKey) throws Exception { + int nonce = Randomness.get().nextInt(); + ByteArrayOutputStream cipherTextOutput; + ByteArrayOutputStream plainTextOutput; + int startIndex = randomIntBetween(0, testPlaintextArray.length - length); + // encrypt + try (InputStream cipherInputStream = + GCMPacketsCipherInputStream.getEncryptor(new ByteArrayInputStream(testPlaintextArray, startIndex, length), + secretKey, nonce, bcFipsProvider)) { + cipherTextOutput = readAllInputStream(cipherInputStream, GCMPacketsCipherInputStream.getEncryptionSizeFromPlainSize(length)); + } + //decrypt + try (InputStream plainInputStream = + GCMPacketsCipherInputStream.getDecryptor(new ByteArrayInputStream(cipherTextOutput.toByteArray()), + secretKey, nonce, bcFipsProvider)) { + plainTextOutput = readAllInputStream(plainInputStream, + GCMPacketsCipherInputStream.getDecryptionSizeFromCipherSize(cipherTextOutput.size())); + } + assertTrue(Arrays.equals(plainTextOutput.toByteArray(), 0, length, testPlaintextArray, startIndex, startIndex + length)); + } + + private SecretKey generateSecretKey() throws Exception { + return AccessController.doPrivileged((PrivilegedAction) () -> { + try { + KeyGenerator keyGen = KeyGenerator.getInstance("AES", bcFipsProvider); + keyGen.init(256, SecureRandom.getInstance("DEFAULT", bcFipsProvider)); + return keyGen.generateKey(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + // read "adversarily" in small random pieces + private ByteArrayOutputStream readAllInputStream(InputStream inputStream, long size) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(Math.toIntExact(size)); + byte[] temp = new byte[randomIntBetween(1, size != 0 ? Math.toIntExact(size) : 1)]; + do { + int bytesRead = inputStream.read(temp, 0, randomIntBetween(1, temp.length)); + if (bytesRead == -1) { + break; + } + baos.write(temp, 0, bytesRead); + if (randomBoolean()) { + int singleByte = inputStream.read(); + if (singleByte == -1) { + break; + } + baos.write(singleByte); + } + } while (true); + assertThat(baos.size(), Matchers.is(Math.toIntExact(size))); + return baos; + } +} diff --git a/x-pack/plugin/repository-encrypted/src/test/resources/rest-api-spec/test/repository_encrypted/10_basic.yml b/x-pack/plugin/repository-encrypted/src/test/resources/rest-api-spec/test/repository_encrypted/10_basic.yml new file mode 100644 index 0000000000000..858ba3e21e3ae --- /dev/null +++ b/x-pack/plugin/repository-encrypted/src/test/resources/rest-api-spec/test/repository_encrypted/10_basic.yml @@ -0,0 +1,16 @@ +# Integration tests for repository-encrypted +# +"Plugin repository-encrypted is loaded": + - skip: + reason: "contains is a newly added assertion" + features: contains + - do: + cluster.state: {} + + # Get master node id + - set: { master_node: master } + + - do: + nodes.info: {} + + - contains: { nodes.$master.plugins: { name: repository-encrypted } }