diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java index a3b8108d67f2a..8b35b0e2d71db 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java @@ -206,12 +206,14 @@ public void testHandlingMissingRootLevelSnapshotMetadata() throws Exception { Files.delete(repo.resolve(String.format(Locale.ROOT, BlobStoreRepository.SNAPSHOT_NAME_FORMAT, snapshotToCorrupt.getUUID()))); logger.info("--> strip version information from index-N blob"); - final RepositoryData withoutVersions = new RepositoryData(repositoryData.getGenId(), - repositoryData.getSnapshotIds().stream().collect(Collectors.toMap( - SnapshotId::getUUID, Function.identity())), - repositoryData.getSnapshotIds().stream().collect(Collectors.toMap( - SnapshotId::getUUID, repositoryData::getSnapshotState)), - Collections.emptyMap(), Collections.emptyMap(), ShardGenerations.EMPTY, IndexMetaDataGenerations.EMPTY); + final RepositoryData withoutVersions = new RepositoryData( + repositoryData.getGenId(), + repositoryData.getSnapshotIds().stream().collect(Collectors.toMap(SnapshotId::getUUID, Function.identity())), + repositoryData.getSnapshotIds().stream().collect(Collectors.toMap(SnapshotId::getUUID, repositoryData::getSnapshotState)), + Collections.emptyMap(), + Collections.emptyMap(), + ShardGenerations.EMPTY, + IndexMetaDataGenerations.EMPTY); Files.write(repo.resolve(BlobStoreRepository.INDEX_FILE_PREFIX + withoutVersions.getGenId()), BytesReference.toBytes(BytesReference.bytes( @@ -330,20 +332,18 @@ public void testRepairBrokenShardGenerations() throws Exception { assertFileExists(initialShardMetaPath); Files.move(initialShardMetaPath, shardPath.resolve(BlobStoreRepository.INDEX_FILE_PREFIX + randomIntBetween(1, 1000))); - final RepositoryData repositoryData1 = getRepositoryData(repoName); + final RepositoryData repositoryData = getRepositoryData(repoName); final Map snapshotIds = - repositoryData1.getSnapshotIds().stream().collect(Collectors.toMap(SnapshotId::getUUID, Function.identity())); + repositoryData.getSnapshotIds().stream().collect(Collectors.toMap(SnapshotId::getUUID, Function.identity())); final RepositoryData brokenRepoData = new RepositoryData( - repositoryData1.getGenId(), snapshotIds, snapshotIds.values().stream().collect( - Collectors.toMap(SnapshotId::getUUID, repositoryData1::getSnapshotState)), - snapshotIds.values().stream().collect( - Collectors.toMap(SnapshotId::getUUID, repositoryData1::getVersion)), - repositoryData1.getIndices().values().stream().collect( - Collectors.toMap(Function.identity(), repositoryData1::getSnapshots) - ), ShardGenerations.builder().putAll(repositoryData1.shardGenerations()).put(indexId, 0, "0").build(), - repositoryData1.indexMetaDataGenerations() - ); - Files.write(repoPath.resolve(BlobStoreRepository.INDEX_FILE_PREFIX + repositoryData1.getGenId()), + repositoryData.getGenId(), + snapshotIds, + snapshotIds.values().stream().collect(Collectors.toMap(SnapshotId::getUUID, repositoryData::getSnapshotState)), + snapshotIds.values().stream().collect(Collectors.toMap(SnapshotId::getUUID, repositoryData::getVersion)), + repositoryData.getIndices().values().stream().collect(Collectors.toMap(Function.identity(), repositoryData::getSnapshots)), + ShardGenerations.builder().putAll(repositoryData.shardGenerations()).put(indexId, 0, "0").build(), + repositoryData.indexMetaDataGenerations()); + Files.write(repoPath.resolve(BlobStoreRepository.INDEX_FILE_PREFIX + repositoryData.getGenId()), BytesReference.toBytes(BytesReference.bytes( brokenRepoData.snapshotsToXContent(XContentFactory.jsonBuilder(), Version.CURRENT))), StandardOpenOption.TRUNCATE_EXISTING); diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java index 246f93540ffb4..a734d33775814 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java @@ -22,6 +22,7 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.util.CollectionUtils; @@ -70,8 +71,14 @@ public final class RepositoryData { /** * An instance initialized for an empty repository. */ - public static final RepositoryData EMPTY = new RepositoryData(EMPTY_REPO_GEN, Collections.emptyMap(), Collections.emptyMap(), - Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), ShardGenerations.EMPTY, + public static final RepositoryData EMPTY = new RepositoryData( + EMPTY_REPO_GEN, + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + ShardGenerations.EMPTY, IndexMetaDataGenerations.EMPTY); /** @@ -107,19 +114,34 @@ public final class RepositoryData { */ private final ShardGenerations shardGenerations; - public RepositoryData(long genId, Map snapshotIds, Map snapshotStates, - Map snapshotVersions, Map> indexSnapshots, - ShardGenerations shardGenerations, IndexMetaDataGenerations indexMetaDataGenerations) { - this(genId, Collections.unmodifiableMap(snapshotIds), Collections.unmodifiableMap(snapshotStates), + public RepositoryData( + long genId, + Map snapshotIds, + Map snapshotStates, + Map snapshotVersions, + Map> indexSnapshots, + ShardGenerations shardGenerations, + IndexMetaDataGenerations indexMetaDataGenerations) { + this( + genId, + Collections.unmodifiableMap(snapshotIds), + Collections.unmodifiableMap(snapshotStates), Collections.unmodifiableMap(snapshotVersions), indexSnapshots.keySet().stream().collect(Collectors.toUnmodifiableMap(IndexId::getName, Function.identity())), - Collections.unmodifiableMap(indexSnapshots), shardGenerations, indexMetaDataGenerations); + Collections.unmodifiableMap(indexSnapshots), + shardGenerations, + indexMetaDataGenerations); } - private RepositoryData(long genId, Map snapshotIds, Map snapshotStates, - Map snapshotVersions, Map indices, - Map> indexSnapshots, ShardGenerations shardGenerations, - IndexMetaDataGenerations indexMetaDataGenerations) { + private RepositoryData( + long genId, + Map snapshotIds, + Map snapshotStates, + Map snapshotVersions, + Map indices, + Map> indexSnapshots, + ShardGenerations shardGenerations, + IndexMetaDataGenerations indexMetaDataGenerations) { this.genId = genId; this.snapshotIds = snapshotIds; this.snapshotStates = snapshotStates; @@ -129,14 +151,21 @@ private RepositoryData(long genId, Map snapshotIds, Map Set.copyOf(snapshotIdList).size() != snapshotIdList.size()) : "Found duplicate snapshot ids per index in [" + indexSnapshots + "]"; } protected RepositoryData copy() { return new RepositoryData( - genId, snapshotIds, snapshotStates, snapshotVersions, indexSnapshots, shardGenerations, indexMetaDataGenerations); + genId, + snapshotIds, + snapshotStates, + snapshotVersions, + indices, + indexSnapshots, + shardGenerations, + indexMetaDataGenerations); } /** @@ -151,7 +180,14 @@ public RepositoryData withVersions(Map versions) { final Map newVersions = new HashMap<>(snapshotVersions); versions.forEach((id, version) -> newVersions.put(id.getUUID(), version)); return new RepositoryData( - genId, snapshotIds, snapshotStates, newVersions, indexSnapshots, shardGenerations, indexMetaDataGenerations); + genId, + snapshotIds, + snapshotStates, + newVersions, + indices, + indexSnapshots, + shardGenerations, + indexMetaDataGenerations); } public ShardGenerations shardGenerations() { @@ -221,8 +257,7 @@ public List indicesToUpdateAfterRemovingSnapshot(Collection /** * Returns a map of {@link IndexId} to a collection of {@link String} containing all the {@link IndexId} and the - * {@link org.elasticsearch.cluster.metadata.IndexMetadata} blob name in it that can be removed after removing the given snapshot from - * the repository. + * {@link IndexMetadata} blob name in it that can be removed after removing the given snapshot from the repository. * NOTE: Does not return a mapping for {@link IndexId} values that will be removed completely from the repository. * * @param snapshotIds SnapshotIds to remove @@ -255,7 +290,7 @@ public Map> indexMetaDataToRemoveAfterRemovingSnapsh * generations indexed by the shard id they correspond to must be supplied. * @param indexMetaBlobs Map of index metadata blob uuids * @param newIdentifiers Map of new index metadata blob uuids keyed by the identifiers of the - * {@link org.elasticsearch.cluster.metadata.IndexMetadata} in them + * {@link IndexMetadata} in them */ public RepositoryData addSnapshot(final SnapshotId snapshotId, final SnapshotState snapshotState, @@ -298,9 +333,14 @@ public RepositoryData addSnapshot(final SnapshotId snapshotId, newIndexMetaGenerations = indexMetaDataGenerations.withAddedSnapshot(snapshotId, indexMetaBlobs, newIdentifiers); } - return new RepositoryData(genId, snapshots, newSnapshotStates, newSnapshotVersions, allIndexSnapshots, - ShardGenerations.builder().putAll(this.shardGenerations).putAll(shardGenerations).build(), - newIndexMetaGenerations); + return new RepositoryData( + genId, + snapshots, + newSnapshotStates, + newSnapshotVersions, + allIndexSnapshots, + ShardGenerations.builder().putAll(this.shardGenerations).putAll(shardGenerations).build(), + newIndexMetaGenerations); } /** @@ -313,7 +353,14 @@ public RepositoryData withGenId(long newGeneration) { if (newGeneration == genId) { return this; } - return new RepositoryData(newGeneration, snapshotIds, snapshotStates, snapshotVersions, indices, indexSnapshots, shardGenerations, + return new RepositoryData( + newGeneration, + snapshotIds, + snapshotStates, + snapshotVersions, + indices, + indexSnapshots, + shardGenerations, indexMetaDataGenerations); } @@ -354,10 +401,15 @@ public RepositoryData removeSnapshots(final Collection snapshots, fi } } - return new RepositoryData(genId, newSnapshotIds, newSnapshotStates, newSnapshotVersions, indexSnapshots, - ShardGenerations.builder().putAll(shardGenerations).putAll(updatedShardGenerations) - .retainIndicesAndPruneDeletes(indexSnapshots.keySet()).build(), - indexMetaDataGenerations.withRemovedSnapshots(snapshots) + return new RepositoryData( + genId, + newSnapshotIds, + newSnapshotStates, + newSnapshotVersions, + indexSnapshots, + ShardGenerations.builder().putAll(shardGenerations).putAll(updatedShardGenerations) + .retainIndicesAndPruneDeletes(indexSnapshots.keySet()).build(), + indexMetaDataGenerations.withRemovedSnapshots(snapshots) ); } @@ -455,6 +507,7 @@ public List resolveNewIndices(List indicesToResolve, Map> indices = randomIndices(snapshotIds); - RepositoryData newRepoData = new RepositoryData(repositoryData.getGenId(), snapshotIds, snapshotStates, snapshotVersions, indices, - ShardGenerations.EMPTY, IndexMetaDataGenerations.EMPTY); + RepositoryData newRepoData = new RepositoryData( + repositoryData.getGenId(), + snapshotIds, + snapshotStates, + snapshotVersions, + indices, + ShardGenerations.EMPTY, + IndexMetaDataGenerations.EMPTY); List expected = new ArrayList<>(repositoryData.getSnapshotIds()); Collections.sort(expected); List actual = new ArrayList<>(newRepoData.getSnapshotIds()); @@ -228,8 +240,14 @@ public void testIndexThatReferencesAnUnknownSnapshot() throws IOException { } assertNotNull(corruptedIndexId); - RepositoryData corruptedRepositoryData = new RepositoryData(parsedRepositoryData.getGenId(), snapshotIds, snapshotStates, - snapshotVersions, indexSnapshots, shardGenBuilder.build(), IndexMetaDataGenerations.EMPTY); + RepositoryData corruptedRepositoryData = new RepositoryData( + parsedRepositoryData.getGenId(), + snapshotIds, + snapshotStates, + snapshotVersions, + indexSnapshots, + shardGenBuilder.build(), + IndexMetaDataGenerations.EMPTY); final XContentBuilder corruptedBuilder = XContentBuilder.builder(xContent); corruptedRepositoryData.snapshotsToXContent(corruptedBuilder, Version.CURRENT); diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java index 87da696cc6229..063ab23fd924b 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -200,13 +200,13 @@ public void testRepositoryDataConcurrentModificationNotAllowed() { // write to index generational file RepositoryData repositoryData = generateRandomRepoData(); final long startingGeneration = repositoryData.getGenId(); - final PlainActionFuture future1 = PlainActionFuture.newFuture(); - repository.writeIndexGen(repositoryData, startingGeneration, Version.CURRENT, Function.identity(),future1); + final PlainActionFuture future = PlainActionFuture.newFuture(); + repository.writeIndexGen(repositoryData, startingGeneration, Version.CURRENT, Function.identity(), future); // write repo data again to index generational file, errors because we already wrote to the // N+1 generation from which this repository data instance was created - expectThrows(RepositoryException.class, - () -> writeIndexGen(repository, repositoryData.withGenId(startingGeneration + 1), repositoryData.getGenId())); + final RepositoryData fresherRepositoryData = repositoryData.withGenId(startingGeneration + 1); + expectThrows(RepositoryException.class, () -> writeIndexGen(repository, fresherRepositoryData, repositoryData.getGenId())); } public void testBadChunksize() throws Exception { diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java index 66a84840d7bb8..704c04ade3703 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java @@ -92,8 +92,14 @@ public IndexMetadata getSnapshotIndexMetaData(RepositoryData repositoryData, Sna @Override public void getRepositoryData(ActionListener listener) { final IndexId indexId = new IndexId(indexName, "blah"); - listener.onResponse(new RepositoryData(EMPTY_REPO_GEN, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), - Collections.singletonMap(indexId, emptyList()), ShardGenerations.EMPTY, IndexMetaDataGenerations.EMPTY)); + listener.onResponse(new RepositoryData( + EMPTY_REPO_GEN, + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + Collections.singletonMap(indexId, emptyList()), + ShardGenerations.EMPTY, + IndexMetaDataGenerations.EMPTY)); } @Override diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index 4aa6027cd0fb5..0706c90b13816 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -256,8 +256,14 @@ public void getRepositoryData(ActionListener listener) { Index index = remoteIndices.get(indexName).getIndex(); indexSnapshots.put(new IndexId(indexName, index.getUUID()), Collections.singletonList(snapshotId)); } - return new RepositoryData(1, copiedSnapshotIds, snapshotStates, snapshotVersions, indexSnapshots, - ShardGenerations.EMPTY, IndexMetaDataGenerations.EMPTY); + return new RepositoryData( + 1, + copiedSnapshotIds, + snapshotStates, + snapshotVersions, + indexSnapshots, + ShardGenerations.EMPTY, + IndexMetaDataGenerations.EMPTY); }); }