Encrypted Repository with AES GCM Packets

plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java

@@ -124,7 +124,7 @@ protected AzureBlobStore createBlobStore() {
     }
 
     @Override
-    protected ByteSizeValue chunkSize() {
+    public ByteSizeValue chunkSize() {
         return chunkSize;
     }
 

plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java

@@ -94,7 +94,7 @@ protected GoogleCloudStorageBlobStore createBlobStore() {
     }
 
     @Override
-    protected ByteSizeValue chunkSize() {
+    public ByteSizeValue chunkSize() {
         return chunkSize;
     }
 

plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java

@@ -233,7 +233,7 @@ protected HdfsBlobStore createBlobStore() {
     }
 
     @Override
-    protected ByteSizeValue chunkSize() {
+    public ByteSizeValue chunkSize() {
         return chunkSize;
     }
 }

plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java

@@ -207,7 +207,7 @@ protected BlobStore getBlobStore() {
     }
 
     @Override
-    protected ByteSizeValue chunkSize() {
+    public ByteSizeValue chunkSize() {
         return chunkSize;
     }
 }

server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java

@@ -200,6 +200,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     protected final ChecksumBlobStoreFormat<SnapshotInfo> snapshotFormat;
 
+    private final NamedXContentRegistry namedXContentRegistry;
+
     private final boolean readOnly;
 
     private final ChecksumBlobStoreFormat<BlobStoreIndexShardSnapshot> indexShardSnapshotFormat;
@@ -234,6 +236,7 @@ protected BlobStoreRepository(
         restoreRateLimiter = getRateLimiter(metadata.settings(), "max_restore_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB));
         readOnly = metadata.settings().getAsBoolean("readonly", false);
         this.basePath = basePath;
+        this.namedXContentRegistry = namedXContentRegistry;
 
         indexShardSnapshotFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_CODEC, SNAPSHOT_NAME_FORMAT,
             BlobStoreIndexShardSnapshot::fromXContent, namedXContentRegistry, compress);

server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java

@@ -109,7 +109,7 @@ protected BlobStore createBlobStore() throws Exception {
     }
 
     @Override
-    protected ByteSizeValue chunkSize() {
+    public ByteSizeValue chunkSize() {
         return chunkSize;
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java

@@ -76,7 +76,6 @@
 import java.time.Clock;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -310,12 +309,6 @@ public static Path resolveConfigFile(Environment env, String name) {
         return config;
     }
 
-    @Override
-    public Map<String, Repository.Factory> getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry,
-                                                           ClusterService clusterService) {
-        return Collections.singletonMap("source", SourceOnlySnapshotRepository.newRepositoryFactory());
-    }
-
     @Override
     public Optional<EngineFactory> getEngineFactory(IndexSettings indexSettings) {
         if (indexSettings.getValue(SourceOnlySnapshotRepository.SOURCE_ONLY)) {

x-pack/plugin/repository-encrypted/build.gradle

@@ -0,0 +1,31 @@
+evaluationDependsOn(xpackModule('core'))
+
+apply plugin: 'elasticsearch.esplugin'
+esplugin {
+    name 'repository-encrypted'
+    description 'Elasticsearch Expanded Pack Plugin - client-side encrypted repositories.'
+    classname 'org.elasticsearch.repositories.encrypted.EncryptedRepositoryPlugin'
+    extendedPlugins = ['x-pack-core']
+}
+
+thirdPartyAudit {
+    ignoreViolations (
+      'org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider$CoreSecureRandom',
+      'org.bouncycastle.jcajce.provider.ProvSunTLSKDF',
+      'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$BaseTLSKeyGeneratorSpi',
+      'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSKeyMaterialGenerator',
+      'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSKeyMaterialGenerator$2',
+      'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSMasterSecretGenerator',
+      'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSMasterSecretGenerator$2',
+      'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSPRFKeyGenerator',
+      'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSRsaPreMasterSecretGenerator',
+      'org.bouncycastle.jcajce.provider.ProvSunTLSKDF$TLSRsaPreMasterSecretGenerator$2',
+    )
+}
+
+dependencies {
+  compile "org.bouncycastle:bc-fips:1.0.1"
+  compile "org.bouncycastle:bcpkix-fips:1.0.3"
+}
+
+integTest.enabled = false

x-pack/plugin/repository-encrypted/licenses/bc-fips-1.0.1.jar.sha1

@@ -0,0 +1 @@
+ed8dd3144761eaa33b9c56f5e2bef85f1b731d6f

x-pack/plugin/repository-encrypted/licenses/bc-fips-LICENSE.txt

@@ -0,0 +1,12 @@
+Copyright (c) 2000 - 2019 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
+following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

x-pack/plugin/repository-encrypted/licenses/bcpkix-fips-1.0.3.jar.sha1

@@ -0,0 +1 @@
+33c47b105777c9dcc8a08188186bd35401366bd1

x-pack/plugin/repository-encrypted/licenses/bcpkix-fips-LICENSE.txt

@@ -0,0 +1,12 @@
+Copyright (c) 2000 - 2019 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
+following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/BlobEncryptionMetadata.java

@@ -0,0 +1,180 @@
+package org.elasticsearch.repositories.encrypted;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+public class BlobEncryptionMetadata {
+
+    private final int maxPacketSizeInBytes;
+    private final int authTagSizeInBytes;
+    private final int ivSizeInBytes;
+    private final byte[] dataEncryptionKeyMaterial;
+    private List<PacketInfo> packetsInfoList;
+
+    public BlobEncryptionMetadata(int maxPacketSizeInBytes, int ivSizeInBytes, int authTagSizeInBytes,
+                                  byte[] dataEncryptionKeyMaterial, List<PacketInfo> packetsInfoList) {
+        this.maxPacketSizeInBytes = maxPacketSizeInBytes;
+        this.ivSizeInBytes = ivSizeInBytes;
+        this.authTagSizeInBytes = authTagSizeInBytes;
+        this.dataEncryptionKeyMaterial = dataEncryptionKeyMaterial;
+        // consistency check of the packet infos
+        for (PacketInfo packetInfo : packetsInfoList) {
+            if (packetInfo.getSizeInBytes() > maxPacketSizeInBytes) {
+                throw new IllegalArgumentException();
+            }
+            if (packetInfo.getIv().length != ivSizeInBytes) {
+                throw new IllegalArgumentException();
+            }
+            if (packetInfo.getAuthTag().length != authTagSizeInBytes) {
+                throw new IllegalArgumentException();
+            }
+        }
+        this.packetsInfoList = Collections.unmodifiableList(packetsInfoList);
+    }
+
+    public BlobEncryptionMetadata(InputStream inputStream) throws IOException {
+        this.maxPacketSizeInBytes = readInt(inputStream);
+        this.authTagSizeInBytes = readInt(inputStream);
+        this.ivSizeInBytes = readInt(inputStream);
+
+        int dataEncryptionKeySizeInBytes = readInt(inputStream);
+        this.dataEncryptionKeyMaterial = readExactlyNBytes(inputStream, dataEncryptionKeySizeInBytes);
+
+        int packetsInfoListSize = readInt(inputStream);
+        List<PacketInfo> packetsInfo = new ArrayList<>(packetsInfoListSize);
+        for (int i = 0; i < packetsInfoListSize; i++) {
+            PacketInfo packetInfo = new PacketInfo(inputStream, ivSizeInBytes, authTagSizeInBytes);
+            // consistency check of the packet infos
+            if (packetInfo.getSizeInBytes() > this.maxPacketSizeInBytes) {
+                throw new IllegalArgumentException();
+            }
+            if (packetInfo.getIv().length != this.ivSizeInBytes) {
+                throw new IllegalArgumentException();
+            }
+            if (packetInfo.getAuthTag().length != this.authTagSizeInBytes) {
+                throw new IllegalArgumentException();
+            }
+            packetsInfo.add(packetInfo);
+        }
+        this.packetsInfoList = Collections.unmodifiableList(packetsInfo);
+    }
+
+    public byte[] getDataEncryptionKeyMaterial() {
+        return dataEncryptionKeyMaterial;
+    }
+
+    public int getMaxPacketSizeInBytes() {
+        return maxPacketSizeInBytes;
+    }
+
+    public int getAuthTagSizeInBytes() {
+        return authTagSizeInBytes;
+    }
+
+    public int getIvSizeInBytes() {
+        return ivSizeInBytes;
+    }
+
+    public List<PacketInfo> getPacketsInfoList() {
+        return packetsInfoList;
+    }
+
+    public void write(OutputStream out) throws IOException {
+        writeInt(out, maxPacketSizeInBytes);
+        writeInt(out, authTagSizeInBytes);
+        writeInt(out, ivSizeInBytes);
+
+        writeInt(out, dataEncryptionKeyMaterial.length);
+        out.write(dataEncryptionKeyMaterial);
+
+        writeInt(out, packetsInfoList.size());
+        for (PacketInfo packetInfo : packetsInfoList) {
+            packetInfo.write(out);
+        }
+    }
+
+    private static int readInt(InputStream inputStream) throws IOException {
+        return ((inputStream.read() & 0xFF) << 24) |
+               ((inputStream.read() & 0xFF) << 16) |
+               ((inputStream.read() & 0xFF) << 8 ) |
+               ((inputStream.read() & 0xFF) << 0 );
+    }
+
+    private static void writeInt(OutputStream out, int val) throws IOException {
+        out.write(val >>> 24);
+        out.write(val >>> 16);
+        out.write(val >>> 8);
+        out.write(val);
+    }
+
+    private static byte[] readExactlyNBytes(InputStream inputStream, int nBytes) throws IOException {
+        byte[] ans = new byte[nBytes];
+        if (nBytes != inputStream.readNBytes(ans, 0, nBytes)) {
+            throw new IOException("Fewer than [" + nBytes + "] read");
+        }
+        return ans;
+    }
+
+    static class PacketInfo {
+
+        private final byte[] iv;
+        private final byte[] authTag;
+        private final int sizeInBytes;
+
+        PacketInfo(byte[] iv, byte[] authTag, int sizeInBytes) {
+            this.iv = iv;
+            this.authTag = authTag;
+            this.sizeInBytes = sizeInBytes;
+        }
+
+        PacketInfo(InputStream inputStream, int ivSizeInBytes, int authTagSizeInBytes) throws IOException {
+            this.iv = readExactlyNBytes(inputStream, ivSizeInBytes);
+            this.authTag = readExactlyNBytes(inputStream, authTagSizeInBytes);
+            this.sizeInBytes = readInt(inputStream);
+        }
+
+        byte[] getIv() {
+            return iv;
+        }
+
+        byte[] getAuthTag() {
+            return authTag;
+        }
+
+        int getSizeInBytes() {
+            return sizeInBytes;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            PacketInfo that = (PacketInfo) o;
+            return sizeInBytes == that.sizeInBytes &&
+                    Arrays.equals(iv, that.iv) &&
+                    Arrays.equals(authTag, that.authTag);
+        }
+
+        @Override
+        public int hashCode() {
+            int result = Objects.hash(sizeInBytes);
+            result = 31 * result + Arrays.hashCode(iv);
+            result = 31 * result + Arrays.hashCode(authTag);
+            return result;
+        }
+
+        public void write(OutputStream out) throws IOException {
+            out.write(iv);
+            out.write(authTag);
+            writeInt(out, sizeInBytes);
+        }
+
+    }
+
+}