Introduce repository UUIDs (#67899)

Today a snapshot repository does not have a well-defined identity. It
can be reregistered with a different cluster under a different name, and
can even be registered with multiple clusters in readonly mode.

This presents problems for cases where we need to refer to a specific
snapshot in a globally-unique fashion. Today we rely on the repository
being registered under the same name on every cluster, but this is not a
safe assumption.

This commit adds a UUID that can be used to uniquely identify a
repository. The UUID is stored in the top-level index blob, represented
by `RepositoryData`, and is also usually copied into the
`RepositoryMetadata` that represents the repository in the cluster
state. The repository UUID is exposed in the get-repositories API; other
more meaningful consumers will be added in due course.

Backport of #67829

docs/reference/snapshot-restore/apis/get-repo-api.asciidoc

@@ -124,9 +124,11 @@ The API returns the following response:
 {
   "my_repository" : {
     "type" : "fs",
+    "uuid" : "0JLknrXbSUiVPuLakHjBrQ",
     "settings" : {
       "location" : "my_backup_location"
     }
   }
 }
 ----
+// TESTRESPONSE[s/"uuid" : "0JLknrXbSUiVPuLakHjBrQ"/"uuid" : $body.my_repository.uuid/]

docs/reference/snapshot-restore/register-repository.asciidoc

@@ -51,12 +51,14 @@ This request returns the following response:
 {
   "my_backup": {
     "type": "fs",
+    "uuid": "0JLknrXbSUiVPuLakHjBrQ",
     "settings": {
       "location": "my_backup_location"
     }
   }
 }
 -----------------------------------
+// TESTRESPONSE[s/"uuid": "0JLknrXbSUiVPuLakHjBrQ"/"uuid": $body.my_backup.uuid/]
 
 To retrieve information about multiple repositories, specify a comma-delimited
 list of repositories. You can also use a wildcard (`*`) when

docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java

@@ -108,6 +108,28 @@ public void waitForRequirements() throws Exception {
         }
     }
 
+    @Before
+    public void populateSnapshotRepository() throws IOException {
+        // The repository UUID is only created on the first write to the repo, so it may or may not exist when running the tests. However to
+        // include the output from the put-repository and get-repositories APIs in the docs we must be sure whether the UUID is returned or
+        // not, so we prepare by taking a snapshot first to ensure that the UUID really has been created.
+        super.initClient();
+
+        final Request putRepoRequest = new Request("PUT", "/_snapshot/test_setup_repo");
+        putRepoRequest.setJsonEntity("{\"type\":\"fs\",\"settings\":{\"location\":\"my_backup_location\"}}");
+        assertOK(adminClient().performRequest(putRepoRequest));
+
+        final Request putSnapshotRequest = new Request("PUT", "/_snapshot/test_setup_repo/test_setup_snap");
+        putSnapshotRequest.addParameter("wait_for_completion", "true");
+        assertOK(adminClient().performRequest(putSnapshotRequest));
+
+        final Request deleteSnapshotRequest = new Request("DELETE", "/_snapshot/test_setup_repo/test_setup_snap");
+        assertOK(adminClient().performRequest(deleteSnapshotRequest));
+
+        final Request deleteRepoRequest = new Request("DELETE", "/_snapshot/test_setup_repo");
+        assertOK(adminClient().performRequest(deleteRepoRequest));
+    }
+
     @After
     public void cleanup() throws Exception {
         if (isMachineLearningTest() || isTransformTest()) {

plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java

@@ -158,7 +158,7 @@ public void testEnforcedCooldownPeriod() throws IOException {
         final BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(repoName);
         final RepositoryData repositoryData = getRepositoryData(repository);
         final RepositoryData modifiedRepositoryData = repositoryData.withVersions(Collections.singletonMap(fakeOldSnapshot,
-            SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION.minimumCompatibilityVersion()));
+            SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION.minimumCompatibilityVersion())).withoutUuid();
         final BytesReference serialized = BytesReference.bytes(modifiedRepositoryData.snapshotsToXContent(XContentFactory.jsonBuilder(),
             SnapshotsService.OLD_SNAPSHOT_FORMAT));
         PlainActionFuture.get(f -> repository.threadPool().generic().execute(ActionRunnable.run(f, () ->

qa/repository-multi-version/src/test/java/org/elasticsearch/upgrades/MultiVersionRepositoryAccessIT.java

@@ -218,7 +218,7 @@ public void testUpgradeMovesRepoToNewMetaVersion() throws IOException {
                     ensureSnapshotRestoreWorks(repoName, "snapshot-2", shards);
                 }
             } else {
-                if (SnapshotsService.useIndexGenerations(minimumNodeVersion()) == false) {
+                if (SnapshotsService.includesRepositoryUuid(minimumNodeVersion()) == false) {
                     assertThat(TEST_STEP, is(TestStep.STEP3_OLD_CLUSTER));
                     final List<Class<? extends Exception>> expectedExceptions =
                         Arrays.asList(ResponseException.class, ElasticsearchStatusException.class);

rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get_repository/20_repository_uuid.yml

@@ -0,0 +1,72 @@
+---
+"Get repository returns UUID":
+  - skip:
+      version: " - 7.11.99"
+      reason:  repository UUIDs introduced in 7.12.0
+
+  - do:
+      snapshot.create_repository:
+        repository: test_repo_uuid_1
+        body:
+          type: fs
+          settings:
+            location: "test_repo_uuid_1_loc"
+
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            number_of_shards:   1
+            number_of_replicas: 0
+
+  - do:
+      snapshot.create:
+        repository: test_repo_uuid_1
+        snapshot: test_snapshot
+        wait_for_completion: true
+
+  - do:
+      snapshot.get_repository: {}
+
+  - match: { test_repo_uuid_1.type: fs }
+  - is_true: test_repo_uuid_1.uuid
+  - set: { test_repo_uuid_1.uuid: repo_uuid }
+  - match: { test_repo_uuid_1.settings.location: "test_repo_uuid_1_loc" }
+  - is_false: test_repo_uuid_1.generation
+  - is_false: test_repo_uuid_1.pending_generation
+
+  - do:
+      snapshot.delete_repository:
+        repository: test_repo_uuid_1
+
+  - do:
+      snapshot.create_repository:
+        repository: test_repo_uuid_1_copy
+        body:
+          type: fs
+          settings:
+            location: "test_repo_uuid_1_loc"
+
+  - do:
+      snapshot.get_repository: {}
+
+  - match: { test_repo_uuid_1_copy.uuid: $repo_uuid }
+
+  - do:
+      snapshot.delete_repository:
+        repository: test_repo_uuid_1_copy
+
+  - do:
+      snapshot.create_repository:
+        repository: test_repo_uuid_1_ro
+        body:
+          type: fs
+          settings:
+            location: "test_repo_uuid_1_loc"
+            read_only: true
+
+  - do:
+      snapshot.get_repository: {}
+
+  - match: { test_repo_uuid_1_ro.uuid: $repo_uuid }

server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java

@@ -260,6 +260,7 @@ public void testHandlingMissingRootLevelSnapshotMetadata() throws Exception {
 
         logger.info("--> strip version information from index-N blob");
         final RepositoryData withoutVersions = new RepositoryData(
+                RepositoryData.MISSING_UUID, // old-format repository data has no UUID
                 repositoryData.getGenId(),
                 repositoryData.getSnapshotIds().stream().collect(Collectors.toMap(SnapshotId::getUUID, Function.identity())),
                 repositoryData.getSnapshotIds().stream().collect(Collectors.toMap(SnapshotId::getUUID, repositoryData::getSnapshotState)),
@@ -270,7 +271,7 @@ public void testHandlingMissingRootLevelSnapshotMetadata() throws Exception {
 
         Files.write(repo.resolve(BlobStoreRepository.INDEX_FILE_PREFIX + withoutVersions.getGenId()),
             BytesReference.toBytes(BytesReference.bytes(
-                withoutVersions.snapshotsToXContent(XContentFactory.jsonBuilder(), Version.CURRENT))),
+                withoutVersions.snapshotsToXContent(XContentFactory.jsonBuilder(), Version.CURRENT, true))),
             StandardOpenOption.TRUNCATE_EXISTING);
 
         logger.info("--> verify that repo is assumed in old metadata format");
@@ -324,8 +325,10 @@ public void testMountCorruptedRepositoryData() throws Exception {
         expectThrows(RepositoryException.class, () -> getRepositoryData(repository));
 
         final String otherRepoName = "other-repo";
-        createRepository(otherRepoName, "fs", Settings.builder()
-            .put("location", repo).put("compress", false));
+        assertAcked(clusterAdmin().preparePutRepository(otherRepoName)
+            .setType("fs")
+            .setVerify(false) // don't try and load the repo data, since it is corrupt
+            .setSettings(Settings.builder().put("location", repo).put("compress", false)));
         final Repository otherRepo = getRepositoryOnMaster(otherRepoName);
 
         logger.info("--> verify loading repository data from newly mounted repository throws RepositoryException");
@@ -389,6 +392,7 @@ public void testRepairBrokenShardGenerations() throws Exception {
         final Map<String, SnapshotId> snapshotIds =
                 repositoryData.getSnapshotIds().stream().collect(Collectors.toMap(SnapshotId::getUUID, Function.identity()));
         final RepositoryData brokenRepoData = new RepositoryData(
+                repositoryData.getUuid(),
                 repositoryData.getGenId(),
                 snapshotIds,
                 snapshotIds.values().stream().collect(Collectors.toMap(SnapshotId::getUUID, repositoryData::getSnapshotState)),

server/src/internalClusterTest/java/org/elasticsearch/snapshots/MultiClusterRepoAccessIT.java

@@ -41,6 +41,8 @@
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
 
 public class MultiClusterRepoAccessIT extends AbstractSnapshotIntegTestCase {
 
@@ -83,8 +85,6 @@ public void testConcurrentDeleteFromOtherCluster() throws InterruptedException {
 
         secondCluster.startMasterOnlyNode();
         secondCluster.startDataOnlyNode();
-        secondCluster.client().admin().cluster().preparePutRepository(repoNameOnSecondCluster).setType("fs")
-                .setSettings(Settings.builder().put("location", repoPath)).get();
 
         createIndexWithRandomDocs("test-idx-1", randomIntBetween(1, 100));
         createFullSnapshot(repoNameOnFirstCluster, "snap-1");
@@ -93,6 +93,8 @@ public void testConcurrentDeleteFromOtherCluster() throws InterruptedException {
         createIndexWithRandomDocs("test-idx-3", randomIntBetween(1, 100));
         createFullSnapshot(repoNameOnFirstCluster, "snap-3");
 
+        secondCluster.client().admin().cluster().preparePutRepository(repoNameOnSecondCluster).setType("fs")
+                .setSettings(Settings.builder().put("location", repoPath)).get();
         secondCluster.client().admin().cluster().prepareDeleteSnapshot(repoNameOnSecondCluster, "snap-1").get();
         secondCluster.client().admin().cluster().prepareDeleteSnapshot(repoNameOnSecondCluster, "snap-2").get();
 
@@ -107,4 +109,38 @@ public void testConcurrentDeleteFromOtherCluster() throws InterruptedException {
         createRepository(repoNameOnFirstCluster, "fs", repoPath);
         createFullSnapshot(repoNameOnFirstCluster, "snap-5");
     }
+
+    @SuppressWarnings("OptionalGetWithoutIsPresent") // we want it to throw if absent
+    public void testConcurrentWipeAndRecreateFromOtherCluster() throws InterruptedException, IOException {
+        internalCluster().startMasterOnlyNode();
+        internalCluster().startDataOnlyNode();
+        final String repoName = "test-repo";
+        createRepository(repoName, "fs", repoPath);
+
+        createIndexWithRandomDocs("test-idx-1", randomIntBetween(1, 100));
+        createFullSnapshot(repoName, "snap-1");
+        final String repoUuid = client().admin().cluster().prepareGetRepositories(repoName).get().repositories()
+                .stream().filter(r -> r.name().equals(repoName)).findFirst().get().uuid();
+
+        secondCluster.startMasterOnlyNode();
+        secondCluster.startDataOnlyNode();
+        assertAcked(secondCluster.client().admin().cluster().preparePutRepository(repoName)
+                .setType("fs")
+                .setSettings(Settings.builder().put("location", repoPath).put("read_only", true)));
+        assertThat(secondCluster.client().admin().cluster().prepareGetRepositories(repoName).get().repositories()
+                .stream().filter(r -> r.name().equals(repoName)).findFirst().get().uuid(), equalTo(repoUuid));
+
+        assertAcked(client().admin().cluster().prepareDeleteRepository(repoName));
+        IOUtils.rm(internalCluster().getCurrentMasterNodeInstance(Environment.class).resolveRepoFile(repoPath.toString()));
+        createRepository(repoName, "fs", repoPath);
+        createFullSnapshot(repoName, "snap-1");
+
+        final String newRepoUuid = client().admin().cluster().prepareGetRepositories(repoName).get().repositories()
+                .stream().filter(r -> r.name().equals(repoName)).findFirst().get().uuid();
+        assertThat(newRepoUuid, not(equalTo((repoUuid))));
+
+        secondCluster.client().admin().cluster().prepareGetSnapshots(repoName).get(); // force another read of the repo data
+        assertThat(secondCluster.client().admin().cluster().prepareGetRepositories(repoName).get().repositories()
+                .stream().filter(r -> r.name().equals(repoName)).findFirst().get().uuid(), equalTo(newRepoUuid));
+    }
 }

server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStatusApisIT.java

@@ -267,7 +267,7 @@ public void testCorrectCountsForDoneShards() throws Exception {
 
     public void testSnapshotStatusOnFailedSnapshot() throws Exception {
         String repoName = "test-repo";
-        createRepository(repoName, "fs");
+        createRepositoryNoVerify(repoName, "fs"); // mustn't load the repository data before we inject the broken snapshot
         final String snapshot = "test-snap-1";
         addBwCFailedSnapshot(repoName, snapshot, Collections.emptyMap());
 

server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetadata.java

@@ -39,6 +39,7 @@
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.List;
+import java.util.function.UnaryOperator;
 
 /**
  * Contains metadata about registered snapshot repositories
@@ -75,6 +76,21 @@ public RepositoriesMetadata(List<RepositoryMetadata> repositories) {
      * @return new instance with updated generations
      */
     public RepositoriesMetadata withUpdatedGeneration(String repoName, long safeGeneration, long pendingGeneration) {
+        return withUpdate(repoName, repositoryMetadata -> new RepositoryMetadata(repositoryMetadata, safeGeneration, pendingGeneration));
+    }
+
+    /**
+     * Creates a new instance that records the UUID of the given repository.
+     *
+     * @param repoName          repository name
+     * @param uuid              repository uuid
+     * @return new instance with updated uuid
+     */
+    public RepositoriesMetadata withUuid(String repoName, String uuid) {
+        return withUpdate(repoName, repositoryMetadata -> repositoryMetadata.withUuid(uuid));
+    }
+
+    private RepositoriesMetadata withUpdate(String repoName, UnaryOperator<RepositoryMetadata> update) {
         int indexOfRepo = -1;
         for (int i = 0; i < repositories.size(); i++) {
             if (repositories.get(i).name().equals(repoName)) {
@@ -86,7 +102,7 @@ public RepositoriesMetadata withUpdatedGeneration(String repoName, long safeGene
             throw new IllegalArgumentException("Unknown repository [" + repoName + "]");
         }
         final List<RepositoryMetadata> updatedRepos = new ArrayList<>(repositories);
-        updatedRepos.set(indexOfRepo, new RepositoryMetadata(repositories.get(indexOfRepo), safeGeneration, pendingGeneration));
+        updatedRepos.set(indexOfRepo, update.apply(repositories.get(indexOfRepo)));
         return new RepositoriesMetadata(updatedRepos);
     }
 
@@ -190,14 +206,20 @@ public static RepositoriesMetadata fromXContent(XContentParser parser) throws IO
                 if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
                     throw new ElasticsearchParseException("failed to parse repository [{}], expected object", name);
                 }
+                String uuid = RepositoryData.MISSING_UUID;
                 String type = null;
                 Settings settings = Settings.EMPTY;
                 long generation = RepositoryData.UNKNOWN_REPO_GEN;
                 long pendingGeneration = RepositoryData.EMPTY_REPO_GEN;
                 while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                     if (token == XContentParser.Token.FIELD_NAME) {
                         String currentFieldName = parser.currentName();
-                        if ("type".equals(currentFieldName)) {
+                        if ("uuid".equals(currentFieldName)) {
+                            if (parser.nextToken() != XContentParser.Token.VALUE_STRING) {
+                                throw new ElasticsearchParseException("failed to parse repository [{}], uuid not a string", name);
+                            }
+                            uuid = parser.text();
+                        } else if ("type".equals(currentFieldName)) {
                             if (parser.nextToken() != XContentParser.Token.VALUE_STRING) {
                                 throw new ElasticsearchParseException("failed to parse repository [{}], unknown type", name);
                             }
@@ -228,7 +250,7 @@ public static RepositoriesMetadata fromXContent(XContentParser parser) throws IO
                 if (type == null) {
                     throw new ElasticsearchParseException("failed to parse repository [{}], missing repository type", name);
                 }
-                repository.add(new RepositoryMetadata(name, type, settings, generation, pendingGeneration));
+                repository.add(new RepositoryMetadata(name, uuid, type, settings, generation, pendingGeneration));
             } else {
                 throw new ElasticsearchParseException("failed to parse repositories");
             }
@@ -262,6 +284,9 @@ public EnumSet<Metadata.XContentContext> context() {
     public static void toXContent(RepositoryMetadata repository, XContentBuilder builder, ToXContent.Params params) throws IOException {
         builder.startObject(repository.name());
         builder.field("type", repository.type());
+        if (repository.uuid().equals(RepositoryData.MISSING_UUID) == false) {
+            builder.field("uuid", repository.uuid());
+        }
         builder.startObject("settings");
         repository.settings().toXContent(builder, params);
         builder.endObject();