Encrypted blob store repository

plugins/repository-azure/build.gradle

@@ -30,6 +30,8 @@ dependencies {
   compile 'com.google.guava:guava:20.0'
   compile 'org.apache.commons:commons-lang3:3.4'
   testCompile project(':test:fixtures:azure-fixture')
+  // required by the test for the encrypted Azure repository
+  testCompile project(path: ':x-pack:plugin:repository-encrypted', configuration: 'testArtifacts')
 }
 
 dependencyLicenses {

plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java

@@ -74,17 +74,20 @@ protected HttpHandler createErroneousHttpHandler(final HttpHandler delegate) {
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
+        final String endpoint = "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=" + httpServerUrl();
+        return Settings.builder()
+                .put(super.nodeSettings(nodeOrdinal))
+                .put(AzureStorageSettings.ENDPOINT_SUFFIX_SETTING.getConcreteSettingForNamespace("test").getKey(), endpoint)
+                .setSecureSettings(nodeSecureSettings(nodeOrdinal))
+                .build();
+    }
+
+    protected MockSecureSettings nodeSecureSettings(int nodeOrdinal) {
+        MockSecureSettings secureSettings = new MockSecureSettings();
         final String key = Base64.getEncoder().encodeToString(randomAlphaOfLength(10).getBytes(StandardCharsets.UTF_8));
-        final MockSecureSettings secureSettings = new MockSecureSettings();
         secureSettings.setString(AzureStorageSettings.ACCOUNT_SETTING.getConcreteSettingForNamespace("test").getKey(), "account");
         secureSettings.setString(AzureStorageSettings.KEY_SETTING.getConcreteSettingForNamespace("test").getKey(), key);
-
-        final String endpoint = "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=" + httpServerUrl();
-        return Settings.builder()
-            .put(super.nodeSettings(nodeOrdinal))
-            .put(AzureStorageSettings.ENDPOINT_SUFFIX_SETTING.getConcreteSettingForNamespace("test").getKey(), endpoint)
-            .setSecureSettings(secureSettings)
-            .build();
+        return secureSettings;
     }
 
     /**

plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/EncryptedAzureBlobStoreRepositoryIntegTests.java

@@ -0,0 +1,110 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.repositories.azure;
+
+import org.elasticsearch.common.blobstore.BlobMetaData;
+import org.elasticsearch.common.settings.MockSecureSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.license.License;
+import org.elasticsearch.license.LicenseService;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
+import org.elasticsearch.repositories.encrypted.DecryptionPacketsInputStream;
+import org.elasticsearch.repositories.encrypted.EncryptedRepository;
+import org.elasticsearch.repositories.encrypted.EncryptedRepositoryPlugin;
+import org.elasticsearch.repositories.encrypted.LocalStateEncryptedRepositoryPlugin;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.BeforeClass;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+public class EncryptedAzureBlobStoreRepositoryIntegTests extends AzureBlobStoreRepositoryTests {
+
+    private static List<String> repositoryNames;
+
+    @BeforeClass
+    private static void preGenerateRepositoryNames() {
+        List<String> names = new ArrayList<>();
+        for (int i = 0; i < 32; i++) {
+            names.add("test-repo-" + i);
+        }
+        repositoryNames = Collections.synchronizedList(names);
+    }
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder()
+                .put(super.nodeSettings(nodeOrdinal))
+                .put(LicenseService.SELF_GENERATED_LICENSE_TYPE.getKey(), License.LicenseType.TRIAL.getTypeName())
+                .build();
+    }
+
+    @Override
+    protected MockSecureSettings nodeSecureSettings(int nodeOrdinal) {
+        MockSecureSettings secureSettings = super.nodeSecureSettings(nodeOrdinal);
+        for (String repositoryName : repositoryNames) {
+            secureSettings.setString(EncryptedRepositoryPlugin.ENCRYPTION_PASSWORD_SETTING.
+                    getConcreteSettingForNamespace(repositoryName).getKey(), "password" + repositoryName);
+        }
+        return secureSettings;
+    }
+
+    @Override
+    protected String randomRepositoryName() {
+        return repositoryNames.remove(randomIntBetween(0, repositoryNames.size() - 1));
+    }
+
+    protected long blobLengthFromDiskLength(BlobMetaData blobMetaData) {
+        if (BlobStoreRepository.INDEX_LATEST_BLOB.equals(blobMetaData.name())) {
+            // index.latest is not encrypted, hence the size on disk is equal to the content
+            return blobMetaData.length();
+        } else {
+            return DecryptionPacketsInputStream.getDecryptionLength(blobMetaData.length() -
+                    EncryptedRepository.MetadataIdentifier.byteLength(), EncryptedRepository.PACKET_LENGTH_IN_BYTES);
+        }
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(LocalStateEncryptedRepositoryPlugin.class, TestAzureRepositoryPlugin.class);
+    }
+
+    @Override
+    protected String repositoryType() {
+        return EncryptedRepositoryPlugin.REPOSITORY_TYPE_NAME;
+    }
+
+    @Override
+    protected Settings repositorySettings() {
+        final Settings.Builder settings = Settings.builder();
+        settings.put(super.repositorySettings());
+        settings.put(EncryptedRepositoryPlugin.DELEGATE_TYPE.getKey(), AzureRepository.TYPE);
+        if (ESTestCase.randomBoolean()) {
+            long size = 1 << ESTestCase.randomInt(10);
+            settings.put("chunk_size", new ByteSizeValue(size, ByteSizeUnit.KB));
+        }
+        return settings.build();
+    }
+}

plugins/repository-gcs/build.gradle

@@ -54,6 +54,8 @@ dependencies {
   compile 'com.google.apis:google-api-services-storage:v1-rev20190426-1.28.0'
 
   testCompile project(':test:fixtures:gcs-fixture')
+  // required by the test for the encrypted GCS repository
+  testCompile project(path: ':x-pack:plugin:repository-encrypted', configuration: 'testArtifacts')
 }
 
 dependencyLicenses {

plugins/repository-gcs/qa/google-cloud-storage/build.gradle

@@ -34,6 +34,9 @@ apply plugin: 'elasticsearch.test.fixtures'
 // TODO think about flattening qa:google-cloud-storage project into parent
 dependencies {
   testCompile project(path: ':plugins:repository-gcs')
+  // required by the third-party test for the encrypted GCS repository
+  testCompile project(path: ':x-pack:plugin:repository-encrypted')
+  testCompile project(path: ':x-pack:plugin:core')
 }
 
 testFixtures.useFixture(':test:fixtures:gcs-fixture')
@@ -101,6 +104,7 @@ task thirdPartyTest(type: Test) {
   }
 
   include '**/GoogleCloudStorageThirdPartyTests.class'
+  include '**/EncryptedGoogleCloudStorageThirdPartyTests.class'
   systemProperty 'tests.security.manager', false
   systemProperty 'test.google.bucket', gcsBucket
   systemProperty 'test.google.base', gcsBasePath + "_third_party_tests_" + BuildParams.testSeed
@@ -126,10 +130,14 @@ processTestResources {
 
 integTest {
   dependsOn project(':plugins:repository-gcs').bundlePlugin
+  dependsOn project(':x-pack:plugin:core').bundlePlugin
+  dependsOn project(':x-pack:plugin:repository-encrypted').bundlePlugin
 }
 
 testClusters.integTest {
   plugin file(project(':plugins:repository-gcs').bundlePlugin.archiveFile)
+  plugin file(project(':x-pack:plugin:core').bundlePlugin.archiveFile)
+  plugin file(project(':x-pack:plugin:repository-encrypted').bundlePlugin.archiveFile)
 
   keystore 'gcs.client.integration_test.credentials_file', serviceAccountFile, IGNORE_VALUE
 

plugins/repository-gcs/qa/google-cloud-storage/src/test/java/org/elasticsearch/repositories/gcs/EncryptedGoogleCloudStorageThirdPartyTests.java

@@ -0,0 +1,111 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.repositories.gcs;
+
+import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.blobstore.BlobMetaData;
+import org.elasticsearch.common.blobstore.BlobPath;
+import org.elasticsearch.common.settings.MockSecureSettings;
+import org.elasticsearch.common.settings.SecureSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.encrypted.EncryptedRepository;
+import org.elasticsearch.repositories.encrypted.EncryptedRepositoryPlugin;
+import org.elasticsearch.xpack.core.XPackPlugin;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class EncryptedGoogleCloudStorageThirdPartyTests extends GoogleCloudStorageThirdPartyTests {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(XPackPlugin.class, EncryptedRepositoryPlugin.class, GoogleCloudStoragePlugin.class);
+    }
+
+    @Override
+    protected Settings nodeSettings() {
+        return Settings.builder()
+                .put(super.nodeSettings())
+                .put("xpack.license.self_generated.type", "trial")
+                .build();
+    }
+
+    @Override
+    protected SecureSettings credentials() {
+        MockSecureSettings secureSettings = (MockSecureSettings) super.credentials();
+        secureSettings.setString(EncryptedRepositoryPlugin.ENCRYPTION_PASSWORD_SETTING.
+                getConcreteSettingForNamespace("test-encrypted-repo").getKey(), "password-test-repo");
+        return secureSettings;
+    }
+
+    @Override
+    protected void createRepository(final String repoName) {
+        AcknowledgedResponse putRepositoryResponse = client().admin().cluster()
+                .preparePutRepository("test-encrypted-repo")
+                .setType("encrypted")
+                .setSettings(Settings.builder()
+                        .put("delegate_type", "gcs")
+                        .put("bucket", System.getProperty("test.google.bucket"))
+                        .put("base_path", System.getProperty("test.google.base", "")
+                                + "/" + EncryptedGoogleCloudStorageThirdPartyTests.class.getName() )
+                ).get();
+        assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
+    }
+
+    @Override
+    protected void assertCleanupResponse(CleanupRepositoryResponse response, long bytes, long blobs) {
+        // TODO cleanup of root blobs does not count the encryption metadata blobs, but the cleanup of blob containers ("indices" folder)
+        //  does count them; ideally there should be consistency, one way or the other
+        assertThat(response.result().blobs(), equalTo(1L + 2L + 1L /* one metadata blob */));
+        // the cleanup stats of the encrypted repository currently includes only some of the metadata blobs (as per above), which are
+        // themselves cumbersome to size; but the bytes count is stable
+        assertThat(response.result().bytes(), equalTo(244L));
+    }
+
+    @Override
+    protected void assertBlobsByPrefix(BlobPath path, String prefix, Map<String, BlobMetaData> blobs) throws Exception {
+        // blobs are larger after encryption
+        Map<String, BlobMetaData> blobsWithSizeAfterEncryption = new HashMap<>();
+        blobs.forEach((name, meta) -> {
+            blobsWithSizeAfterEncryption.put(name, new BlobMetaData() {
+                @Override
+                public String name() {
+                    return meta.name();
+                }
+
+                @Override
+                public long length() {
+                    return EncryptedRepository.getEncryptedBlobByteLength(meta.length());
+                }
+            });
+        });
+        super.assertBlobsByPrefix(path, prefix, blobsWithSizeAfterEncryption);
+    }
+
+    @Override
+    protected String getTestRepoName() {
+        return "test-encrypted-repo";
+    }
+
+}

plugins/repository-gcs/qa/google-cloud-storage/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java

@@ -69,11 +69,11 @@ protected SecureSettings credentials() {
 
     @Override
     protected void createRepository(final String repoName) {
-        AcknowledgedResponse putRepositoryResponse = client().admin().cluster().preparePutRepository("test-repo")
+        AcknowledgedResponse putRepositoryResponse = client().admin().cluster().preparePutRepository(repoName)
             .setType("gcs")
             .setSettings(Settings.builder()
                 .put("bucket", System.getProperty("test.google.bucket"))
-                .put("base_path", System.getProperty("test.google.base", "/"))
+                .put("base_path", System.getProperty("test.google.base", "") + "/" + GoogleCloudStorageThirdPartyTests.class.getName() )
             ).get();
         assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
     }