From 57ab96828914e31025430381f6d067edbeadfc29 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 25 Jan 2021 17:49:03 +0000 Subject: [PATCH] Introduce repository UUIDs (#67899) Today a snapshot repository does not have a well-defined identity. It can be reregistered with a different cluster under a different name, and can even be registered with multiple clusters in readonly mode. This presents problems for cases where we need to refer to a specific snapshot in a globally-unique fashion. Today we rely on the repository being registered under the same name on every cluster, but this is not a safe assumption. This commit adds a UUID that can be used to uniquely identify a repository. The UUID is stored in the top-level index blob, represented by `RepositoryData`, and is also usually copied into the `RepositoryMetadata` that represents the repository in the cluster state. The repository UUID is exposed in the get-repositories API; other more meaningful consumers will be added in due course. Backport of #67829 --- .../apis/get-repo-api.asciidoc | 2 + .../register-repository.asciidoc | 2 + .../smoketest/DocsClientYamlTestSuiteIT.java | 22 ++++ .../s3/S3BlobStoreRepositoryTests.java | 2 +- .../MultiVersionRepositoryAccessIT.java | 2 +- .../20_repository_uuid.yml | 72 +++++++++++ .../CorruptedBlobStoreRepositoryIT.java | 10 +- .../snapshots/MultiClusterRepoAccessIT.java | 40 +++++- .../snapshots/SnapshotStatusApisIT.java | 2 +- .../metadata/RepositoriesMetadata.java | 31 ++++- .../cluster/metadata/RepositoryMetadata.java | 42 +++++- .../repositories/RepositoriesService.java | 121 +++++++++++++++--- .../repositories/RepositoryData.java | 97 +++++++++++++- .../blobstore/BlobStoreRepository.java | 48 +++++-- .../snapshots/SnapshotsService.java | 12 ++ .../repositories/RepositoryDataTests.java | 8 +- .../blobstore/BlobStoreRepositoryTests.java | 1 + ...epositoriesMetadataSerializationTests.java | 9 +- .../index/shard/RestoreOnlyRepository.java | 2 + .../AbstractSnapshotIntegTestCase.java | 19 ++- .../xpack/ccr/repository/CcrRepository.java | 2 + .../slm/SLMSnapshotBlockingIntegTests.java | 2 +- ...ryptedFSBlobStoreRepositoryIntegTests.java | 39 ++++-- 23 files changed, 526 insertions(+), 61 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get_repository/20_repository_uuid.yml diff --git a/docs/reference/snapshot-restore/apis/get-repo-api.asciidoc b/docs/reference/snapshot-restore/apis/get-repo-api.asciidoc index 3ea15aa8793e7..441bf3e8e400a 100644 --- a/docs/reference/snapshot-restore/apis/get-repo-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/get-repo-api.asciidoc @@ -124,9 +124,11 @@ The API returns the following response: { "my_repository" : { "type" : "fs", + "uuid" : "0JLknrXbSUiVPuLakHjBrQ", "settings" : { "location" : "my_backup_location" } } } ---- +// TESTRESPONSE[s/"uuid" : "0JLknrXbSUiVPuLakHjBrQ"/"uuid" : $body.my_repository.uuid/] diff --git a/docs/reference/snapshot-restore/register-repository.asciidoc b/docs/reference/snapshot-restore/register-repository.asciidoc index 1f210326d225a..d3924a0940d34 100644 --- a/docs/reference/snapshot-restore/register-repository.asciidoc +++ b/docs/reference/snapshot-restore/register-repository.asciidoc @@ -51,12 +51,14 @@ This request returns the following response: { "my_backup": { "type": "fs", + "uuid": "0JLknrXbSUiVPuLakHjBrQ", "settings": { "location": "my_backup_location" } } } ----------------------------------- +// TESTRESPONSE[s/"uuid": "0JLknrXbSUiVPuLakHjBrQ"/"uuid": $body.my_backup.uuid/] To retrieve information about multiple repositories, specify a comma-delimited list of repositories. You can also use a wildcard (`*`) when diff --git a/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java b/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java index 838b084967522..3db07b96610d4 100644 --- a/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java +++ b/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java @@ -108,6 +108,28 @@ public void waitForRequirements() throws Exception { } } + @Before + public void populateSnapshotRepository() throws IOException { + // The repository UUID is only created on the first write to the repo, so it may or may not exist when running the tests. However to + // include the output from the put-repository and get-repositories APIs in the docs we must be sure whether the UUID is returned or + // not, so we prepare by taking a snapshot first to ensure that the UUID really has been created. + super.initClient(); + + final Request putRepoRequest = new Request("PUT", "/_snapshot/test_setup_repo"); + putRepoRequest.setJsonEntity("{\"type\":\"fs\",\"settings\":{\"location\":\"my_backup_location\"}}"); + assertOK(adminClient().performRequest(putRepoRequest)); + + final Request putSnapshotRequest = new Request("PUT", "/_snapshot/test_setup_repo/test_setup_snap"); + putSnapshotRequest.addParameter("wait_for_completion", "true"); + assertOK(adminClient().performRequest(putSnapshotRequest)); + + final Request deleteSnapshotRequest = new Request("DELETE", "/_snapshot/test_setup_repo/test_setup_snap"); + assertOK(adminClient().performRequest(deleteSnapshotRequest)); + + final Request deleteRepoRequest = new Request("DELETE", "/_snapshot/test_setup_repo"); + assertOK(adminClient().performRequest(deleteRepoRequest)); + } + @After public void cleanup() throws Exception { if (isMachineLearningTest() || isTransformTest()) { diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java index 4a8f308ce1bba..bb3222fa080c7 100644 --- a/plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -158,7 +158,7 @@ public void testEnforcedCooldownPeriod() throws IOException { final BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(repoName); final RepositoryData repositoryData = getRepositoryData(repository); final RepositoryData modifiedRepositoryData = repositoryData.withVersions(Collections.singletonMap(fakeOldSnapshot, - SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION.minimumCompatibilityVersion())); + SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION.minimumCompatibilityVersion())).withoutUuid(); final BytesReference serialized = BytesReference.bytes(modifiedRepositoryData.snapshotsToXContent(XContentFactory.jsonBuilder(), SnapshotsService.OLD_SNAPSHOT_FORMAT)); PlainActionFuture.get(f -> repository.threadPool().generic().execute(ActionRunnable.run(f, () -> diff --git a/qa/repository-multi-version/src/test/java/org/elasticsearch/upgrades/MultiVersionRepositoryAccessIT.java b/qa/repository-multi-version/src/test/java/org/elasticsearch/upgrades/MultiVersionRepositoryAccessIT.java index 616b19345c241..caba391ae9921 100644 --- a/qa/repository-multi-version/src/test/java/org/elasticsearch/upgrades/MultiVersionRepositoryAccessIT.java +++ b/qa/repository-multi-version/src/test/java/org/elasticsearch/upgrades/MultiVersionRepositoryAccessIT.java @@ -218,7 +218,7 @@ public void testUpgradeMovesRepoToNewMetaVersion() throws IOException { ensureSnapshotRestoreWorks(repoName, "snapshot-2", shards); } } else { - if (SnapshotsService.useIndexGenerations(minimumNodeVersion()) == false) { + if (SnapshotsService.includesRepositoryUuid(minimumNodeVersion()) == false) { assertThat(TEST_STEP, is(TestStep.STEP3_OLD_CLUSTER)); final List> expectedExceptions = Arrays.asList(ResponseException.class, ElasticsearchStatusException.class); diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get_repository/20_repository_uuid.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get_repository/20_repository_uuid.yml new file mode 100644 index 0000000000000..936ca5f7b81b6 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get_repository/20_repository_uuid.yml @@ -0,0 +1,72 @@ +--- +"Get repository returns UUID": + - skip: + version: " - 7.11.99" + reason: repository UUIDs introduced in 7.12.0 + + - do: + snapshot.create_repository: + repository: test_repo_uuid_1 + body: + type: fs + settings: + location: "test_repo_uuid_1_loc" + + - do: + indices.create: + index: test_index + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + + - do: + snapshot.create: + repository: test_repo_uuid_1 + snapshot: test_snapshot + wait_for_completion: true + + - do: + snapshot.get_repository: {} + + - match: { test_repo_uuid_1.type: fs } + - is_true: test_repo_uuid_1.uuid + - set: { test_repo_uuid_1.uuid: repo_uuid } + - match: { test_repo_uuid_1.settings.location: "test_repo_uuid_1_loc" } + - is_false: test_repo_uuid_1.generation + - is_false: test_repo_uuid_1.pending_generation + + - do: + snapshot.delete_repository: + repository: test_repo_uuid_1 + + - do: + snapshot.create_repository: + repository: test_repo_uuid_1_copy + body: + type: fs + settings: + location: "test_repo_uuid_1_loc" + + - do: + snapshot.get_repository: {} + + - match: { test_repo_uuid_1_copy.uuid: $repo_uuid } + + - do: + snapshot.delete_repository: + repository: test_repo_uuid_1_copy + + - do: + snapshot.create_repository: + repository: test_repo_uuid_1_ro + body: + type: fs + settings: + location: "test_repo_uuid_1_loc" + read_only: true + + - do: + snapshot.get_repository: {} + + - match: { test_repo_uuid_1_ro.uuid: $repo_uuid } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java index 4062e9f8df75d..823e29a6a5265 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java @@ -260,6 +260,7 @@ public void testHandlingMissingRootLevelSnapshotMetadata() throws Exception { logger.info("--> strip version information from index-N blob"); final RepositoryData withoutVersions = new RepositoryData( + RepositoryData.MISSING_UUID, // old-format repository data has no UUID repositoryData.getGenId(), repositoryData.getSnapshotIds().stream().collect(Collectors.toMap(SnapshotId::getUUID, Function.identity())), repositoryData.getSnapshotIds().stream().collect(Collectors.toMap(SnapshotId::getUUID, repositoryData::getSnapshotState)), @@ -270,7 +271,7 @@ public void testHandlingMissingRootLevelSnapshotMetadata() throws Exception { Files.write(repo.resolve(BlobStoreRepository.INDEX_FILE_PREFIX + withoutVersions.getGenId()), BytesReference.toBytes(BytesReference.bytes( - withoutVersions.snapshotsToXContent(XContentFactory.jsonBuilder(), Version.CURRENT))), + withoutVersions.snapshotsToXContent(XContentFactory.jsonBuilder(), Version.CURRENT, true))), StandardOpenOption.TRUNCATE_EXISTING); logger.info("--> verify that repo is assumed in old metadata format"); @@ -324,8 +325,10 @@ public void testMountCorruptedRepositoryData() throws Exception { expectThrows(RepositoryException.class, () -> getRepositoryData(repository)); final String otherRepoName = "other-repo"; - createRepository(otherRepoName, "fs", Settings.builder() - .put("location", repo).put("compress", false)); + assertAcked(clusterAdmin().preparePutRepository(otherRepoName) + .setType("fs") + .setVerify(false) // don't try and load the repo data, since it is corrupt + .setSettings(Settings.builder().put("location", repo).put("compress", false))); final Repository otherRepo = getRepositoryOnMaster(otherRepoName); logger.info("--> verify loading repository data from newly mounted repository throws RepositoryException"); @@ -389,6 +392,7 @@ public void testRepairBrokenShardGenerations() throws Exception { final Map snapshotIds = repositoryData.getSnapshotIds().stream().collect(Collectors.toMap(SnapshotId::getUUID, Function.identity())); final RepositoryData brokenRepoData = new RepositoryData( + repositoryData.getUuid(), repositoryData.getGenId(), snapshotIds, snapshotIds.values().stream().collect(Collectors.toMap(SnapshotId::getUUID, repositoryData::getSnapshotState)), diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MultiClusterRepoAccessIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MultiClusterRepoAccessIT.java index c2574b4c8566f..a38ce59880ad2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MultiClusterRepoAccessIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MultiClusterRepoAccessIT.java @@ -41,6 +41,8 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; public class MultiClusterRepoAccessIT extends AbstractSnapshotIntegTestCase { @@ -83,8 +85,6 @@ public void testConcurrentDeleteFromOtherCluster() throws InterruptedException { secondCluster.startMasterOnlyNode(); secondCluster.startDataOnlyNode(); - secondCluster.client().admin().cluster().preparePutRepository(repoNameOnSecondCluster).setType("fs") - .setSettings(Settings.builder().put("location", repoPath)).get(); createIndexWithRandomDocs("test-idx-1", randomIntBetween(1, 100)); createFullSnapshot(repoNameOnFirstCluster, "snap-1"); @@ -93,6 +93,8 @@ public void testConcurrentDeleteFromOtherCluster() throws InterruptedException { createIndexWithRandomDocs("test-idx-3", randomIntBetween(1, 100)); createFullSnapshot(repoNameOnFirstCluster, "snap-3"); + secondCluster.client().admin().cluster().preparePutRepository(repoNameOnSecondCluster).setType("fs") + .setSettings(Settings.builder().put("location", repoPath)).get(); secondCluster.client().admin().cluster().prepareDeleteSnapshot(repoNameOnSecondCluster, "snap-1").get(); secondCluster.client().admin().cluster().prepareDeleteSnapshot(repoNameOnSecondCluster, "snap-2").get(); @@ -107,4 +109,38 @@ public void testConcurrentDeleteFromOtherCluster() throws InterruptedException { createRepository(repoNameOnFirstCluster, "fs", repoPath); createFullSnapshot(repoNameOnFirstCluster, "snap-5"); } + + @SuppressWarnings("OptionalGetWithoutIsPresent") // we want it to throw if absent + public void testConcurrentWipeAndRecreateFromOtherCluster() throws InterruptedException, IOException { + internalCluster().startMasterOnlyNode(); + internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + createRepository(repoName, "fs", repoPath); + + createIndexWithRandomDocs("test-idx-1", randomIntBetween(1, 100)); + createFullSnapshot(repoName, "snap-1"); + final String repoUuid = client().admin().cluster().prepareGetRepositories(repoName).get().repositories() + .stream().filter(r -> r.name().equals(repoName)).findFirst().get().uuid(); + + secondCluster.startMasterOnlyNode(); + secondCluster.startDataOnlyNode(); + assertAcked(secondCluster.client().admin().cluster().preparePutRepository(repoName) + .setType("fs") + .setSettings(Settings.builder().put("location", repoPath).put("read_only", true))); + assertThat(secondCluster.client().admin().cluster().prepareGetRepositories(repoName).get().repositories() + .stream().filter(r -> r.name().equals(repoName)).findFirst().get().uuid(), equalTo(repoUuid)); + + assertAcked(client().admin().cluster().prepareDeleteRepository(repoName)); + IOUtils.rm(internalCluster().getCurrentMasterNodeInstance(Environment.class).resolveRepoFile(repoPath.toString())); + createRepository(repoName, "fs", repoPath); + createFullSnapshot(repoName, "snap-1"); + + final String newRepoUuid = client().admin().cluster().prepareGetRepositories(repoName).get().repositories() + .stream().filter(r -> r.name().equals(repoName)).findFirst().get().uuid(); + assertThat(newRepoUuid, not(equalTo((repoUuid)))); + + secondCluster.client().admin().cluster().prepareGetSnapshots(repoName).get(); // force another read of the repo data + assertThat(secondCluster.client().admin().cluster().prepareGetRepositories(repoName).get().repositories() + .stream().filter(r -> r.name().equals(repoName)).findFirst().get().uuid(), equalTo(newRepoUuid)); + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStatusApisIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStatusApisIT.java index 2c3cb1b56df5c..a73795cf676b7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStatusApisIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStatusApisIT.java @@ -267,7 +267,7 @@ public void testCorrectCountsForDoneShards() throws Exception { public void testSnapshotStatusOnFailedSnapshot() throws Exception { String repoName = "test-repo"; - createRepository(repoName, "fs"); + createRepositoryNoVerify(repoName, "fs"); // mustn't load the repository data before we inject the broken snapshot final String snapshot = "test-snap-1"; addBwCFailedSnapshot(repoName, snapshot, Collections.emptyMap()); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetadata.java index c4e3478f4cf5e..2f4dc87e91747 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetadata.java @@ -39,6 +39,7 @@ import java.util.Collections; import java.util.EnumSet; import java.util.List; +import java.util.function.UnaryOperator; /** * Contains metadata about registered snapshot repositories @@ -75,6 +76,21 @@ public RepositoriesMetadata(List repositories) { * @return new instance with updated generations */ public RepositoriesMetadata withUpdatedGeneration(String repoName, long safeGeneration, long pendingGeneration) { + return withUpdate(repoName, repositoryMetadata -> new RepositoryMetadata(repositoryMetadata, safeGeneration, pendingGeneration)); + } + + /** + * Creates a new instance that records the UUID of the given repository. + * + * @param repoName repository name + * @param uuid repository uuid + * @return new instance with updated uuid + */ + public RepositoriesMetadata withUuid(String repoName, String uuid) { + return withUpdate(repoName, repositoryMetadata -> repositoryMetadata.withUuid(uuid)); + } + + private RepositoriesMetadata withUpdate(String repoName, UnaryOperator update) { int indexOfRepo = -1; for (int i = 0; i < repositories.size(); i++) { if (repositories.get(i).name().equals(repoName)) { @@ -86,7 +102,7 @@ public RepositoriesMetadata withUpdatedGeneration(String repoName, long safeGene throw new IllegalArgumentException("Unknown repository [" + repoName + "]"); } final List updatedRepos = new ArrayList<>(repositories); - updatedRepos.set(indexOfRepo, new RepositoryMetadata(repositories.get(indexOfRepo), safeGeneration, pendingGeneration)); + updatedRepos.set(indexOfRepo, update.apply(repositories.get(indexOfRepo))); return new RepositoriesMetadata(updatedRepos); } @@ -190,6 +206,7 @@ public static RepositoriesMetadata fromXContent(XContentParser parser) throws IO if (parser.nextToken() != XContentParser.Token.START_OBJECT) { throw new ElasticsearchParseException("failed to parse repository [{}], expected object", name); } + String uuid = RepositoryData.MISSING_UUID; String type = null; Settings settings = Settings.EMPTY; long generation = RepositoryData.UNKNOWN_REPO_GEN; @@ -197,7 +214,12 @@ public static RepositoriesMetadata fromXContent(XContentParser parser) throws IO while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { String currentFieldName = parser.currentName(); - if ("type".equals(currentFieldName)) { + if ("uuid".equals(currentFieldName)) { + if (parser.nextToken() != XContentParser.Token.VALUE_STRING) { + throw new ElasticsearchParseException("failed to parse repository [{}], uuid not a string", name); + } + uuid = parser.text(); + } else if ("type".equals(currentFieldName)) { if (parser.nextToken() != XContentParser.Token.VALUE_STRING) { throw new ElasticsearchParseException("failed to parse repository [{}], unknown type", name); } @@ -228,7 +250,7 @@ public static RepositoriesMetadata fromXContent(XContentParser parser) throws IO if (type == null) { throw new ElasticsearchParseException("failed to parse repository [{}], missing repository type", name); } - repository.add(new RepositoryMetadata(name, type, settings, generation, pendingGeneration)); + repository.add(new RepositoryMetadata(name, uuid, type, settings, generation, pendingGeneration)); } else { throw new ElasticsearchParseException("failed to parse repositories"); } @@ -262,6 +284,9 @@ public EnumSet context() { public static void toXContent(RepositoryMetadata repository, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(repository.name()); builder.field("type", repository.type()); + if (repository.uuid().equals(RepositoryData.MISSING_UUID) == false) { + builder.field("uuid", repository.uuid()); + } builder.startObject("settings"); repository.settings().toXContent(builder, params); builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoryMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoryMetadata.java index 1c5b8a12e78e2..ed2d32924d94c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoryMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoryMetadata.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.repositories.RepositoryData; +import org.elasticsearch.snapshots.SnapshotsService; import java.io.IOException; import java.util.Objects; @@ -36,6 +37,7 @@ public class RepositoryMetadata implements Writeable { public static final Version REPO_GEN_IN_CS_VERSION = Version.V_7_6_0; private final String name; + private final String uuid; private final String type; private final Settings settings; @@ -57,15 +59,16 @@ public class RepositoryMetadata implements Writeable { * @param settings repository settings */ public RepositoryMetadata(String name, String type, Settings settings) { - this(name, type, settings, RepositoryData.UNKNOWN_REPO_GEN, RepositoryData.EMPTY_REPO_GEN); + this(name, RepositoryData.MISSING_UUID, type, settings, RepositoryData.UNKNOWN_REPO_GEN, RepositoryData.EMPTY_REPO_GEN); } public RepositoryMetadata(RepositoryMetadata metadata, long generation, long pendingGeneration) { - this(metadata.name, metadata.type, metadata.settings, generation, pendingGeneration); + this(metadata.name, metadata.uuid, metadata.type, metadata.settings, generation, pendingGeneration); } - public RepositoryMetadata(String name, String type, Settings settings, long generation, long pendingGeneration) { + public RepositoryMetadata(String name, String uuid, String type, Settings settings, long generation, long pendingGeneration) { this.name = name; + this.uuid = uuid; this.type = type; this.settings = settings; this.generation = generation; @@ -92,6 +95,19 @@ public String type() { return this.type; } + /** + * Return the repository UUID, if set and known. The repository UUID is stored in the repository and typically populated here when the + * repository is registered or when we write to it. It may not be set if the repository is maintaining support for versions before + * {@link SnapshotsService#REPOSITORY_UUID_IN_REPO_DATA_VERSION}. It may not be known if the repository was registered with {@code + * ?verify=false} and has had no subsequent writes. Consumers may, if desired, try and fill in a missing value themselves by retrieving + * the {@link RepositoryData} and calling {@link org.elasticsearch.repositories.RepositoriesService#updateRepositoryUuidInMetadata}. + * + * @return repository UUID, or {@link RepositoryData#MISSING_UUID} if the UUID is not set or not known. + */ + public String uuid() { + return this.uuid; + } + /** * Returns repository settings * @@ -127,6 +143,11 @@ public long pendingGeneration() { public RepositoryMetadata(StreamInput in) throws IOException { name = in.readString(); + if (in.getVersion().onOrAfter(SnapshotsService.REPOSITORY_UUID_IN_REPO_DATA_VERSION)) { + uuid = in.readString(); + } else { + uuid = RepositoryData.MISSING_UUID; + } type = in.readString(); settings = Settings.readSettingsFromStream(in); if (in.getVersion().onOrAfter(REPO_GEN_IN_CS_VERSION)) { @@ -146,6 +167,9 @@ public RepositoryMetadata(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(name); + if (out.getVersion().onOrAfter(SnapshotsService.REPOSITORY_UUID_IN_REPO_DATA_VERSION)) { + out.writeString(uuid); + } out.writeString(type); Settings.writeSettingsToStream(settings, out); if (out.getVersion().onOrAfter(REPO_GEN_IN_CS_VERSION)) { @@ -161,7 +185,7 @@ public void writeTo(StreamOutput out) throws IOException { * @return {@code true} if both instances equal in all fields but the generation fields */ public boolean equalsIgnoreGenerations(RepositoryMetadata other) { - return name.equals(other.name) && type.equals(other.type()) && settings.equals(other.settings()); + return name.equals(other.name) && uuid.equals(other.uuid()) && type.equals(other.type()) && settings.equals(other.settings()); } @Override @@ -172,6 +196,7 @@ public boolean equals(Object o) { RepositoryMetadata that = (RepositoryMetadata) o; if (!name.equals(that.name)) return false; + if (!uuid.equals(that.uuid)) return false; if (!type.equals(that.type)) return false; if (generation != that.generation) return false; if (pendingGeneration != that.pendingGeneration) return false; @@ -180,11 +205,16 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(name, type, settings, generation, pendingGeneration); + return Objects.hash(name, uuid, type, settings, generation, pendingGeneration); } @Override public String toString() { - return "RepositoryMetadata{" + name + "}{" + type + "}{" + settings + "}{" + generation + "}{" + pendingGeneration + "}"; + return "RepositoryMetadata{" + name + "}{" + uuid + "}{" + type + "}{" + settings + "}{" + + generation + "}{" + pendingGeneration + "}"; + } + + public RepositoryMetadata withUuid(String uuid) { + return new RepositoryMetadata(name, uuid, type, settings, generation, pendingGeneration); } } diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java index 1f88bf7b1ffe5..b9f7fdc97ce7b 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java @@ -24,6 +24,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; +import org.elasticsearch.action.StepListener; import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest; import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -31,6 +32,7 @@ import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateApplier; +import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.RepositoryCleanupInProgress; import org.elasticsearch.cluster.RestoreInProgress; import org.elasticsearch.cluster.SnapshotDeletionsInProgress; @@ -60,9 +62,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import java.util.Set; /** * Service responsible for maintaining and providing access to snapshot repositories on nodes. @@ -126,30 +128,54 @@ public void registerRepository(final PutRepositoryRequest request, final ActionL final RepositoryMetadata newRepositoryMetadata = new RepositoryMetadata(request.name(), request.type(), request.settings()); validate(request.name()); - final ActionListener registrationListener; + // Trying to create the new repository on master to make sure it works + try { + closeRepository(createRepository(newRepositoryMetadata, typesRegistry)); + } catch (Exception e) { + listener.onFailure(e); + return; + } + + final StepListener acknowledgementStep = new StepListener<>(); + final StepListener publicationStep = new StepListener<>(); + if (request.verify()) { - registrationListener = ActionListener.delegateFailure(listener, (delegatedListener, clusterStateUpdateResponse) -> { + + // When publication has completed (and all acks received or timed out) then verify the repository. + // (if acks timed out then acknowledgementStep completes before the master processes this cluster state, hence why we have + // to wait for the publication to be complete too) + final StepListener> verifyStep = new StepListener<>(); + publicationStep.whenComplete(ignored -> acknowledgementStep.whenComplete(clusterStateUpdateResponse -> { if (clusterStateUpdateResponse.isAcknowledged()) { // The response was acknowledged - all nodes should know about the new repository, let's verify them - verifyRepository(request.name(), delegatedListener.map(discoveryNodes -> clusterStateUpdateResponse)); + verifyRepository(request.name(), verifyStep); } else { - delegatedListener.onResponse(clusterStateUpdateResponse); + verifyStep.onResponse(null); } - }); - } else { - registrationListener = listener; - } + }, listener::onFailure), listener::onFailure); - // Trying to create the new repository on master to make sure it works - try { - closeRepository(createRepository(newRepositoryMetadata, typesRegistry)); - } catch (Exception e) { - registrationListener.onFailure(e); - return; + // When verification has completed, get the repository data for the first time + final StepListener getRepositoryDataStep = new StepListener<>(); + verifyStep.whenComplete(ignored -> threadPool.generic().execute( + ActionRunnable.wrap(getRepositoryDataStep, l -> repository(request.name()).getRepositoryData(l))), listener::onFailure); + + // When the repository metadata is ready, update the repository UUID stored in the cluster state, if available + final StepListener updateRepoUuidStep = new StepListener<>(); + getRepositoryDataStep.whenComplete( + repositoryData -> updateRepositoryUuidInMetadata(clusterService, request.name(), repositoryData, updateRepoUuidStep), + listener::onFailure); + + // Finally respond to the outer listener with the response from the original cluster state update + updateRepoUuidStep.whenComplete( + ignored -> acknowledgementStep.whenComplete(listener::onResponse, listener::onFailure), + listener::onFailure); + + } else { + acknowledgementStep.whenComplete(listener::onResponse, listener::onFailure); } clusterService.submitStateUpdateTask("put_repository [" + request.name() + "]", - new AckedClusterStateUpdateTask(request, registrationListener) { + new AckedClusterStateUpdateTask(request, acknowledgementStep) { @Override public ClusterState execute(ClusterState currentState) { @@ -185,6 +211,7 @@ public ClusterState execute(ClusterState currentState) { @Override public void onFailure(String source, Exception e) { logger.warn(() -> new ParameterizedMessage("failed to create repository [{}]", request.name()), e); + publicationStep.onFailure(e); super.onFailure(source, e); } @@ -193,9 +220,71 @@ public boolean mustAck(DiscoveryNode discoveryNode) { // repository is created on both master and data nodes return discoveryNode.isMasterNode() || discoveryNode.isDataNode(); } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + publicationStep.onResponse(null); + } }); } + /** + * Set the repository UUID in the named repository's {@link RepositoryMetadata} to match the UUID in its {@link RepositoryData}, + * which may involve a cluster state update. + * + * @param listener notified when the {@link RepositoryMetadata} is updated, possibly on this thread or possibly on the master service + * thread + */ + public static void updateRepositoryUuidInMetadata( + ClusterService clusterService, + final String repositoryName, + RepositoryData repositoryData, + ActionListener listener) { + + final String repositoryUuid = repositoryData.getUuid(); + if (repositoryUuid.equals(RepositoryData.MISSING_UUID)) { + listener.onResponse(null); + return; + } + + final RepositoriesMetadata currentReposMetadata + = clusterService.state().metadata().custom(RepositoriesMetadata.TYPE, RepositoriesMetadata.EMPTY); + final RepositoryMetadata repositoryMetadata = currentReposMetadata.repository(repositoryName); + if (repositoryMetadata == null || repositoryMetadata.uuid().equals(repositoryUuid)) { + listener.onResponse(null); + return; + } + + clusterService.submitStateUpdateTask("update repository UUID [" + repositoryName + "] to [" + repositoryUuid + "]", + new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) { + final RepositoriesMetadata currentReposMetadata + = currentState.metadata().custom(RepositoriesMetadata.TYPE, RepositoriesMetadata.EMPTY); + + final RepositoryMetadata repositoryMetadata = currentReposMetadata.repository(repositoryName); + if (repositoryMetadata == null || repositoryMetadata.uuid().equals(repositoryUuid)) { + return currentState; + } else { + final RepositoriesMetadata newReposMetadata = currentReposMetadata.withUuid(repositoryName, repositoryUuid); + final Metadata.Builder metadata + = Metadata.builder(currentState.metadata()).putCustom(RepositoriesMetadata.TYPE, newReposMetadata); + return ClusterState.builder(currentState).metadata(metadata).build(); + } + } + + @Override + public void onFailure(String source, Exception e) { + listener.onFailure(e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + listener.onResponse(null); + } + }); + } + /** * Unregisters repository in the cluster *

diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java index 923908e2cf77c..fac4e6a1c01dc 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java @@ -67,10 +67,16 @@ public final class RepositoryData { */ public static final long CORRUPTED_REPO_GEN = -3L; + /** + * Sentinel value for the repository UUID indicating that it is not set. + */ + public static final String MISSING_UUID = "_na_"; + /** * An instance initialized for an empty repository. */ public static final RepositoryData EMPTY = new RepositoryData( + MISSING_UUID, EMPTY_REPO_GEN, Collections.emptyMap(), Collections.emptyMap(), @@ -80,6 +86,11 @@ public final class RepositoryData { ShardGenerations.EMPTY, IndexMetaDataGenerations.EMPTY); + /** + * A UUID that identifies this repository. + */ + private final String uuid; + /** * The generational id of the index file from which the repository data was read. */ @@ -114,6 +125,7 @@ public final class RepositoryData { private final ShardGenerations shardGenerations; public RepositoryData( + String uuid, long genId, Map snapshotIds, Map snapshotStates, @@ -122,6 +134,7 @@ public RepositoryData( ShardGenerations shardGenerations, IndexMetaDataGenerations indexMetaDataGenerations) { this( + uuid, genId, Collections.unmodifiableMap(snapshotIds), Collections.unmodifiableMap(snapshotStates), @@ -134,6 +147,7 @@ public RepositoryData( } private RepositoryData( + String uuid, long genId, Map snapshotIds, Map snapshotStates, @@ -142,6 +156,7 @@ private RepositoryData( Map> indexSnapshots, ShardGenerations shardGenerations, IndexMetaDataGenerations indexMetaDataGenerations) { + this.uuid = Objects.requireNonNull(uuid); this.genId = genId; this.snapshotIds = snapshotIds; this.snapshotStates = snapshotStates; @@ -158,6 +173,7 @@ private RepositoryData( protected RepositoryData copy() { return new RepositoryData( + uuid, genId, snapshotIds, snapshotStates, @@ -180,6 +196,7 @@ public RepositoryData withVersions(Map versions) { final Map newVersions = new HashMap<>(snapshotVersions); versions.forEach((id, version) -> newVersions.put(id.getUUID(), version)); return new RepositoryData( + uuid, genId, snapshotIds, snapshotStates, @@ -194,6 +211,14 @@ public ShardGenerations shardGenerations() { return shardGenerations; } + /** + * @return The UUID of this repository, or {@link RepositoryData#MISSING_UUID} if this repository has no UUID because it still + * supports access from versions earlier than {@link SnapshotsService#REPOSITORY_UUID_IN_REPO_DATA_VERSION}. + */ + public String getUuid() { + return uuid; + } + /** * Gets the generational index file id from which this instance was read. */ @@ -334,6 +359,7 @@ public RepositoryData addSnapshot(final SnapshotId snapshotId, } return new RepositoryData( + uuid, genId, snapshots, newSnapshotStates, @@ -354,6 +380,7 @@ public RepositoryData withGenId(long newGeneration) { return this; } return new RepositoryData( + uuid, newGeneration, snapshotIds, snapshotStates, @@ -364,6 +391,40 @@ public RepositoryData withGenId(long newGeneration) { indexMetaDataGenerations); } + /** + * Make a copy of this instance with the given UUID and all other fields unchanged. + */ + public RepositoryData withUuid(String uuid) { + assert this.uuid.equals(MISSING_UUID) : this.uuid; + assert uuid.equals(MISSING_UUID) == false; + return new RepositoryData( + uuid, + genId, + snapshotIds, + snapshotStates, + snapshotVersions, + indices, + indexSnapshots, + shardGenerations, + indexMetaDataGenerations); + } + + /** + * For test purposes, make a copy of this instance with the UUID removed and all other fields unchanged, as if from an older version. + */ + public RepositoryData withoutUuid() { + return new RepositoryData( + MISSING_UUID, + genId, + snapshotIds, + snapshotStates, + snapshotVersions, + indices, + indexSnapshots, + shardGenerations, + indexMetaDataGenerations); + } + /** * Remove snapshots and remove any indices that no longer exist in the repository due to the deletion of the snapshots. * @@ -402,6 +463,7 @@ public RepositoryData removeSnapshots(final Collection snapshots, fi } return new RepositoryData( + uuid, genId, newSnapshotIds, newSnapshotStates, @@ -505,10 +567,24 @@ public List resolveNewIndices(List indicesToResolve, Map> indexMetaLookup = new HashMap<>(); Map indexMetaIdentifiers = null; + String uuid = MISSING_UUID; while (parser.nextToken() == XContentParser.Token.FIELD_NAME) { final String field = parser.currentName(); switch (field) { @@ -628,6 +717,11 @@ public static RepositoryData snapshotsFromXContent(XContentParser parser, long g "this snapshot repository format requires Elasticsearch version [" + version + "] or later"); } break; + case UUID: + XContentParserUtils.ensureExpectedToken(XContentParser.Token.VALUE_STRING, parser.nextToken(), parser); + uuid = parser.text(); + assert uuid.equals(MISSING_UUID) == false; + break; default: XContentParserUtils.throwUnknownField(field, parser.getTokenLocation()); } @@ -637,6 +731,7 @@ public static RepositoryData snapshotsFromXContent(XContentParser parser, long g XContentParserUtils.ensureExpectedToken(null, parser.nextToken(), parser); return new RepositoryData( + uuid, genId, snapshots, snapshotStates, diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 75ead529d5c1b..826167b488fc7 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -40,6 +40,7 @@ import org.elasticsearch.action.StepListener; import org.elasticsearch.action.support.GroupedActionListener; import org.elasticsearch.action.support.PlainListenableActionFuture; +import org.elasticsearch.action.support.ThreadedActionListener; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.RepositoryCleanupInProgress; @@ -103,6 +104,7 @@ import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.IndexMetaDataGenerations; +import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryCleanupResult; import org.elasticsearch.repositories.RepositoryData; @@ -147,6 +149,7 @@ import java.util.stream.Stream; import static org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo.canonicalName; +import static org.elasticsearch.repositories.RepositoryData.MISSING_UUID; /** * BlobStore - based implementation of Snapshot Repository @@ -1504,21 +1507,34 @@ private void doGetRepositoryData(ActionListener listener) { } try { final CachedRepositoryData cached = latestKnownRepositoryData.get(); - final RepositoryData loaded; // Caching is not used with #bestEffortConsistency see docs on #cacheRepositoryData for details if (bestEffortConsistency == false && cached.generation() == genToLoad && cached.hasData()) { - loaded = cached.repositoryData(); + listener.onResponse(cached.repositoryData()); } else { - loaded = getRepositoryData(genToLoad); + final RepositoryData loaded = getRepositoryData(genToLoad); if (cached == null || cached.generation() < genToLoad) { // We can cache serialized in the most recent version here without regard to the actual repository metadata version // since we're only caching the information that we just wrote and thus won't accidentally cache any information // that isn't safe cacheRepositoryData(compressRepoDataForCache(BytesReference.bytes( - loaded.snapshotsToXContent(XContentFactory.jsonBuilder(), Version.CURRENT))), genToLoad); + loaded.snapshotsToXContent(XContentFactory.jsonBuilder(), Version.CURRENT, true))), genToLoad); + } + if (loaded.getUuid().equals(metadata.uuid())) { + listener.onResponse(loaded); + } else { + // someone switched the repo contents out from under us + RepositoriesService.updateRepositoryUuidInMetadata( + clusterService, + metadata.name(), + loaded, + new ThreadedActionListener<>( + logger, + threadPool, + ThreadPool.Names.GENERIC, + listener.map(v -> loaded), + false)); } } - listener.onResponse(loaded); return; } catch (RepositoryException e) { // If the generation to load changed concurrently and we didn't just try loading the same generation before we retry @@ -1796,7 +1812,7 @@ public void onFailure(Exception e) { })), listener::onFailure); filterRepositoryDataStep.whenComplete(filteredRepositoryData -> { final long newGen = setPendingStep.result(); - final RepositoryData newRepositoryData = filteredRepositoryData.withGenId(newGen); + final RepositoryData newRepositoryData = updateRepositoryData(filteredRepositoryData, version, newGen); if (latestKnownRepoGen.get() >= newGen) { throw new IllegalArgumentException( "Tried writing generation [" + newGen + "] but repository is at least at generation [" + latestKnownRepoGen.get() @@ -1834,10 +1850,14 @@ public ClusterState execute(ClusterState currentState) { "Tried to update from unexpected pending repo generation [" + meta.pendingGeneration() + "] after write to generation [" + newGen + "]"); } - return updateRepositoryGenerationsIfNecessary(stateFilter.apply(ClusterState.builder(currentState) - .metadata(Metadata.builder(currentState.getMetadata()).putCustom(RepositoriesMetadata.TYPE, - currentState.metadata().custom(RepositoriesMetadata.TYPE) - .withUpdatedGeneration(metadata.name(), newGen, newGen))).build()), expectedGen, newGen); + final RepositoriesMetadata currentMetadata = currentState.metadata().custom(RepositoriesMetadata.TYPE); + final RepositoriesMetadata withGenerations = currentMetadata.withUpdatedGeneration(metadata.name(), newGen, newGen); + final RepositoriesMetadata withUuid = meta.uuid().equals(newRepositoryData.getUuid()) + ? withGenerations + : withGenerations.withUuid(metadata.name(), newRepositoryData.getUuid()); + final ClusterState newClusterState = stateFilter.apply(ClusterState.builder(currentState).metadata( + Metadata.builder(currentState.getMetadata()).putCustom(RepositoriesMetadata.TYPE, withUuid)).build()); + return updateRepositoryGenerationsIfNecessary(newClusterState, expectedGen, newGen); } @Override @@ -1870,6 +1890,14 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS }, listener::onFailure); } + private RepositoryData updateRepositoryData(RepositoryData repositoryData, Version repositoryMetaversion, long newGen) { + if (SnapshotsService.includesRepositoryUuid(repositoryMetaversion) && repositoryData.getUuid().equals(MISSING_UUID)) { + return repositoryData.withGenId(newGen).withUuid(UUIDs.randomBase64UUID()); + } else { + return repositoryData.withGenId(newGen); + } + } + /** * Write {@code index.latest} blob to support using this repository as the basis of a url repository. * diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 2163d97b73975..989a0641a7018 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -139,6 +139,8 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus public static final Version INDEX_GEN_IN_REPO_DATA_VERSION = Version.V_7_9_0; + public static final Version REPOSITORY_UUID_IN_REPO_DATA_VERSION = Version.V_7_12_0; + public static final Version OLD_SNAPSHOT_FORMAT = Version.V_7_5_0; public static final Version MULTI_DELETE_VERSION = Version.V_7_8_0; @@ -2288,6 +2290,16 @@ public static boolean useIndexGenerations(Version repositoryMetaVersion) { return repositoryMetaVersion.onOrAfter(INDEX_GEN_IN_REPO_DATA_VERSION); } + /** + * Checks whether the metadata version supports writing {@link ShardGenerations} to the repository. + * + * @param repositoryMetaVersion version to check + * @return true if version supports {@link ShardGenerations} + */ + public static boolean includesRepositoryUuid(Version repositoryMetaVersion) { + return repositoryMetaVersion.onOrAfter(REPOSITORY_UUID_IN_REPO_DATA_VERSION); + } + /** Deletes snapshot from repository * * @param deleteEntry delete entry in cluster state diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java index f277854aee29e..e082296e1f1ba 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java @@ -46,6 +46,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.repositories.RepositoryData.EMPTY_REPO_GEN; +import static org.elasticsearch.repositories.RepositoryData.MISSING_UUID; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -75,7 +76,7 @@ public void testIndicesToUpdateAfterRemovingSnapshot() { } public void testXContent() throws IOException { - RepositoryData repositoryData = generateRandomRepoData(); + RepositoryData repositoryData = generateRandomRepoData().withUuid(UUIDs.randomBase64UUID()); XContentBuilder builder = JsonXContent.contentBuilder(); repositoryData.snapshotsToXContent(builder, Version.CURRENT); try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { @@ -141,6 +142,7 @@ public void testInitIndices() { snapshotVersions.put(snapshotId.getUUID(), randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion())); } RepositoryData repositoryData = new RepositoryData( + MISSING_UUID, EMPTY_REPO_GEN, snapshotIds, Collections.emptyMap(), @@ -151,6 +153,7 @@ public void testInitIndices() { // test that initializing indices works Map> indices = randomIndices(snapshotIds); RepositoryData newRepoData = new RepositoryData( + repositoryData.getUuid(), repositoryData.getGenId(), snapshotIds, snapshotStates, @@ -203,7 +206,7 @@ public void testGetSnapshotState() { public void testIndexThatReferencesAnUnknownSnapshot() throws IOException { final XContent xContent = randomFrom(XContentType.values()).xContent(); - final RepositoryData repositoryData = generateRandomRepoData(); + final RepositoryData repositoryData = generateRandomRepoData().withUuid(UUIDs.randomBase64UUID()); XContentBuilder builder = XContentBuilder.builder(xContent); repositoryData.snapshotsToXContent(builder, Version.CURRENT); @@ -241,6 +244,7 @@ public void testIndexThatReferencesAnUnknownSnapshot() throws IOException { assertNotNull(corruptedIndexId); RepositoryData corruptedRepositoryData = new RepositoryData( + parsedRepositoryData.getUuid(), parsedRepositoryData.getGenId(), snapshotIds, snapshotStates, diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java index 9c1beed8b6057..0f2d57dbdba20 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -258,6 +258,7 @@ private BlobStoreRepository setupRepo() { client.admin().cluster().preparePutRepository(repositoryName) .setType(REPO_TYPE) .setSettings(Settings.builder().put(node().settings()).put("location", location)) + .setVerify(false) // prevent eager reading of repo data .get(); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); diff --git a/server/src/test/java/org/elasticsearch/snapshots/RepositoriesMetadataSerializationTests.java b/server/src/test/java/org/elasticsearch/snapshots/RepositoriesMetadataSerializationTests.java index 4328daeaf0982..849f0f53262b8 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/RepositoriesMetadataSerializationTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/RepositoriesMetadataSerializationTests.java @@ -44,8 +44,13 @@ protected Custom createTestInstance() { for (int i = 0; i < numberOfRepositories; i++) { // divide by 2 to not overflow when adding to this number for the pending generation below final long generation = randomNonNegativeLong() / 2L; - entries.add(new RepositoryMetadata(randomAlphaOfLength(10), randomAlphaOfLength(10), randomSettings(), generation, - generation + randomLongBetween(0, generation))); + entries.add(new RepositoryMetadata( + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomSettings(), + generation, + generation + randomLongBetween(0, generation))); } entries.sort(Comparator.comparing(RepositoryMetadata::name)); return new RepositoriesMetadata(entries); diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java index 1ae2f4e1ea963..b69b28f6a0b35 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java @@ -49,6 +49,7 @@ import static java.util.Collections.emptyList; import static org.elasticsearch.repositories.RepositoryData.EMPTY_REPO_GEN; +import static org.elasticsearch.repositories.RepositoryData.MISSING_UUID; /** A dummy repository for testing which just needs restore overridden */ public abstract class RestoreOnlyRepository extends AbstractLifecycleComponent implements Repository { @@ -94,6 +95,7 @@ public IndexMetadata getSnapshotIndexMetaData(RepositoryData repositoryData, Sna public void getRepositoryData(ActionListener listener) { final IndexId indexId = new IndexId(indexName, "blah"); listener.onResponse(new RepositoryData( + MISSING_UUID, EMPTY_REPO_GEN, Collections.emptyMap(), Collections.emptyMap(), diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java index 81908fd5a8c0e..026ba8edb0fef 100644 --- a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -155,6 +155,15 @@ protected void disableRepoConsistencyCheck(String reason) { skipRepoConsistencyCheckReason = reason; } + protected RepositoryData getRepositoryData(String repoName, Version version) { + final RepositoryData repositoryData = getRepositoryData(repoName); + if (SnapshotsService.includesRepositoryUuid(version) == false) { + return repositoryData.withoutUuid(); + } else { + return repositoryData; + } + } + protected RepositoryData getRepositoryData(String repository) { return getRepositoryData((Repository) getRepositoryOnMaster(repository)); } @@ -291,6 +300,14 @@ protected void createRepository(String repoName, String type) { createRepository(repoName, type, randomRepositorySettings()); } + protected void createRepositoryNoVerify(String repoName, String type) { + logger.info("--> creating repository [{}] [{}]", repoName, type); + assertAcked(clusterAdmin().preparePutRepository(repoName) + .setVerify(false) + .setType(type) + .setSettings(randomRepositorySettings())); + } + protected Settings.Builder randomRepositorySettings() { final Settings.Builder settings = Settings.builder(); settings.put("location", randomRepoPath()).put("compress", randomBoolean()); @@ -328,7 +345,7 @@ protected String initWithSnapshotVersion(String repoName, Path repoPath, Version assertThat(snapshotInfo.totalShards(), is(0)); logger.info("--> writing downgraded RepositoryData for repository metadata version [{}]", version); - final RepositoryData repositoryData = getRepositoryData(repoName); + final RepositoryData repositoryData = getRepositoryData(repoName, version); final XContentBuilder jsonBuilder = JsonXContent.contentBuilder(); repositoryData.snapshotsToXContent(jsonBuilder, version); final RepositoryData downgradedRepoData = RepositoryData.snapshotsFromXContent(JsonXContent.jsonXContent.createParser( diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index 31509ffcb8835..4c14ab660462c 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -106,6 +106,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.index.seqno.RetentionLeaseActions.RETAIN_ALL; +import static org.elasticsearch.repositories.RepositoryData.MISSING_UUID; import static org.elasticsearch.xpack.ccr.CcrRetentionLeases.retentionLeaseId; import static org.elasticsearch.xpack.ccr.CcrRetentionLeases.syncAddRetentionLease; import static org.elasticsearch.xpack.ccr.CcrRetentionLeases.syncRenewRetentionLease; @@ -261,6 +262,7 @@ public void getRepositoryData(ActionListener listener) { indexSnapshots.put(new IndexId(indexName, index.getUUID()), Collections.singletonList(snapshotId)); } return new RepositoryData( + MISSING_UUID, 1, copiedSnapshotIds, snapshotStates, diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMSnapshotBlockingIntegTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMSnapshotBlockingIntegTests.java index d955fe4fe3070..c643e8da75203 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMSnapshotBlockingIntegTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMSnapshotBlockingIntegTests.java @@ -295,7 +295,7 @@ private void testUnsuccessfulSnapshotRetention(boolean partialSuccess) throws Ex final SnapshotState expectedUnsuccessfulState = partialSuccess ? SnapshotState.PARTIAL : SnapshotState.FAILED; // Setup createAndPopulateIndex(indexName); - createRepository(REPO, "mock"); + createRepositoryNoVerify(REPO, "mock"); createSnapshotPolicy(policyId, "snap", NEVER_EXECUTE_CRON_SCHEDULE, REPO, indexName, true, partialSuccess, new SnapshotRetentionConfiguration(null, 1, 2)); diff --git a/x-pack/plugin/repository-encrypted/src/internalClusterTest/java/org/elasticsearch/repositories/encrypted/EncryptedFSBlobStoreRepositoryIntegTests.java b/x-pack/plugin/repository-encrypted/src/internalClusterTest/java/org/elasticsearch/repositories/encrypted/EncryptedFSBlobStoreRepositoryIntegTests.java index 41815b768b351..0e0766b10ca75 100644 --- a/x-pack/plugin/repository-encrypted/src/internalClusterTest/java/org/elasticsearch/repositories/encrypted/EncryptedFSBlobStoreRepositoryIntegTests.java +++ b/x-pack/plugin/repository-encrypted/src/internalClusterTest/java/org/elasticsearch/repositories/encrypted/EncryptedFSBlobStoreRepositoryIntegTests.java @@ -115,7 +115,6 @@ public void testTamperedEncryptionMetadata() throws Exception { client().admin().cluster().prepareCreateSnapshot(repoName, snapshotName).setWaitForCompletion(true).setIndices("other*").get(); assertAcked(client().admin().cluster().prepareDeleteRepository(repoName)); - createRepository(repoName, Settings.builder().put(repoSettings).put("readonly", randomBoolean()).build(), randomBoolean()); try (Stream rootContents = Files.list(repoPath.resolve(EncryptedRepository.DEK_ROOT_CONTAINER))) { // tamper all DEKs @@ -136,22 +135,40 @@ public void testTamperedEncryptionMetadata() throws Exception { throw new UncheckedIOException(e); } }); - final BlobStoreRepository blobStoreRepository = (BlobStoreRepository) internalCluster().getCurrentMasterNodeInstance( - RepositoriesService.class - ).repository(repoName); - RepositoryException e = expectThrows( + } + + final Settings settings = Settings.builder().put(repoSettings).put("readonly", randomBoolean()).build(); + final boolean verifyOnCreate = randomBoolean(); + + if (verifyOnCreate) { + assertThat( + expectThrows(RepositoryException.class, () -> createRepository(repoName, settings, true)).getMessage(), + containsString("the encryption metadata in the repository has been corrupted") + ); + // it still creates the repository even if verification failed + } else { + createRepository(repoName, settings, false); + } + + final BlobStoreRepository blobStoreRepository = (BlobStoreRepository) internalCluster().getCurrentMasterNodeInstance( + RepositoriesService.class + ).repository(repoName); + assertThat( + expectThrows( RepositoryException.class, () -> PlainActionFuture.get( f -> blobStoreRepository.threadPool().generic().execute(ActionRunnable.wrap(f, blobStoreRepository::getRepositoryData)) ) - ); - assertThat(e.getMessage(), containsString("the encryption metadata in the repository has been corrupted")); - e = expectThrows( + ).getMessage(), + containsString("the encryption metadata in the repository has been corrupted") + ); + assertThat( + expectThrows( RepositoryException.class, () -> client().admin().cluster().prepareRestoreSnapshot(repoName, snapshotName).setWaitForCompletion(true).get() - ); - assertThat(e.getMessage(), containsString("the encryption metadata in the repository has been corrupted")); - } + ).getMessage(), + containsString("the encryption metadata in the repository has been corrupted") + ); } }