From b6fc8c01bcdd600d51c93a9474548ef8249ad1f4 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 10 Sep 2019 09:39:23 +0200 Subject: [PATCH] nicer --- .../cluster/SnapshotsInProgress.java | 7 +-- .../repositories/FilterRepository.java | 2 +- .../repositories/Repository.java | 4 +- .../repositories/RepositoryData.java | 6 +-- .../repositories/ShardGenerations.java | 51 ++++++++++++++++--- .../blobstore/BlobStoreRepository.java | 13 +++-- .../snapshots/SnapshotsService.java | 27 ++-------- .../RepositoriesServiceTests.java | 2 +- .../repositories/RepositoryDataTests.java | 20 ++++---- .../BlobStoreRepositoryRestoreTests.java | 4 +- .../blobstore/BlobStoreRepositoryTests.java | 10 ++-- ...ckEventuallyConsistentRepositoryTests.java | 7 +-- .../index/shard/RestoreOnlyRepository.java | 3 +- .../xpack/ccr/repository/CcrRepository.java | 3 +- .../SourceOnlySnapshotRepository.java | 5 +- .../SourceOnlySnapshotShardTests.java | 5 +- 16 files changed, 98 insertions(+), 71 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java index 424701c98b935..0afd79269796d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java @@ -315,8 +315,8 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; ShardSnapshotStatus status = (ShardSnapshotStatus) o; - return Objects.equals(nodeId, status.nodeId) && Objects.equals(reason, status.reason) && state == status.state; - + return Objects.equals(nodeId, status.nodeId) && Objects.equals(reason, status.reason) + && Objects.equals(generation, status.generation) && state == status.state; } @Override @@ -324,12 +324,13 @@ public int hashCode() { int result = state != null ? state.hashCode() : 0; result = 31 * result + (nodeId != null ? nodeId.hashCode() : 0); result = 31 * result + (reason != null ? reason.hashCode() : 0); + result = 31 * result + (generation != null ? generation.hashCode() : 0); return result; } @Override public String toString() { - return "ShardSnapshotStatus[state=" + state + ", nodeId=" + nodeId + ", reason=" + reason + "]"; + return "ShardSnapshotStatus[state=" + state + ", nodeId=" + nodeId + ", reason=" + reason + ", generation=" + generation + "]"; } } diff --git a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java index 46252d7a762f6..701f476541601 100644 --- a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java @@ -79,7 +79,7 @@ public void initializeSnapshot(SnapshotId snapshotId, List indices, Met } @Override - public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, Map shardGenerations, long startTime, String failure, + public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenerations, long startTime, String failure, int totalShards, List shardFailures, long repositoryStateId, boolean includeGlobalState, MetaData metaData, Map userMetadata, Version version) { diff --git a/server/src/main/java/org/elasticsearch/repositories/Repository.java b/server/src/main/java/org/elasticsearch/repositories/Repository.java index ab7c799a0bc8e..ab6093764502c 100644 --- a/server/src/main/java/org/elasticsearch/repositories/Repository.java +++ b/server/src/main/java/org/elasticsearch/repositories/Repository.java @@ -127,7 +127,7 @@ default Repository create(RepositoryMetaData metaData, Function shardGenerations, long startTime, String failure, + SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenerations, long startTime, String failure, int totalShards, List shardFailures, long repositoryStateId, boolean includeGlobalState, MetaData clusterMetaData, Map userMetadata, Version version); diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java index a690a45bd69ca..408749790fbec 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java @@ -186,7 +186,7 @@ public List indicesAfterRemovingSnapshot(SnapshotId snapshotId) { */ public RepositoryData addSnapshot(final SnapshotId snapshotId, final SnapshotState snapshotState, - final Map shardGenerations) { + final ShardGenerations shardGenerations) { if (snapshotIds.containsKey(snapshotId.getUUID())) { // if the snapshot id already exists in the repository data, it means an old master // that is blocked from the cluster is trying to finalize a snapshot concurrently with @@ -198,7 +198,7 @@ public RepositoryData addSnapshot(final SnapshotId snapshotId, Map newSnapshotStates = new HashMap<>(snapshotStates); newSnapshotStates.put(snapshotId.getUUID(), snapshotState); Map> allIndexSnapshots = new HashMap<>(indexSnapshots); - for (final IndexId indexId : shardGenerations.keySet()) { + for (final IndexId indexId : shardGenerations.indices()) { allIndexSnapshots.computeIfAbsent(indexId, k -> new LinkedHashSet<>()).add(snapshotId); } return new RepositoryData(genId, snapshots, newSnapshotStates, allIndexSnapshots, @@ -228,7 +228,7 @@ public RepositoryData withGenId(long newGeneration) { * Pass {@code null} when this instance should not track shard generations while the cluster still * contains nodes from before {@link SnapshotsService#SHARD_GEN_IN_REPO_DATA_VERSION}. */ - public RepositoryData removeSnapshot(final SnapshotId snapshotId, @Nullable final Map updatedShardGenerations) { + public RepositoryData removeSnapshot(final SnapshotId snapshotId, @Nullable final ShardGenerations updatedShardGenerations) { Map newSnapshotIds = snapshotIds.values().stream() .filter(id -> !snapshotId.equals(id)) .collect(Collectors.toMap(SnapshotId::getUUID, Function.identity())); diff --git a/server/src/main/java/org/elasticsearch/repositories/ShardGenerations.java b/server/src/main/java/org/elasticsearch/repositories/ShardGenerations.java index 5e97e479e88c8..e88fcf46008e2 100644 --- a/server/src/main/java/org/elasticsearch/repositories/ShardGenerations.java +++ b/server/src/main/java/org/elasticsearch/repositories/ShardGenerations.java @@ -19,11 +19,16 @@ package org.elasticsearch.repositories; +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.shard.ShardId; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -31,17 +36,47 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; -final class ShardGenerations implements ToXContent { +public final class ShardGenerations implements ToXContent { public static final ShardGenerations EMPTY = new ShardGenerations(Collections.emptyMap()); private final Map> shardGenerations; - ShardGenerations(Map> shardGenerations) { + public ShardGenerations(Map> shardGenerations) { this.shardGenerations = shardGenerations; } + public static ShardGenerations fromSnapshot(SnapshotsInProgress.Entry snapshot) { + final Map indexLookup = new HashMap<>(); + snapshot.indices().forEach(idx -> indexLookup.put(idx.getName(), idx)); + final Map>> res = new HashMap<>(); + for (final ObjectObjectCursor shard : snapshot.shards()) { + final ShardId shardId = shard.key; + res.computeIfAbsent(indexLookup.get(shardId.getIndexName()), k -> new ArrayList<>()).add(new Tuple<>(shardId, shard.value)); + } + return new ShardGenerations(res.entrySet().stream().collect( + Collectors.toMap(Map.Entry::getKey, entry -> { + final List> status = entry.getValue(); + final String[] gens = new String[ + status.stream().mapToInt(s -> s.v1().getId()) + .max().orElseThrow(() -> new AssertionError("0-shard index is impossible")) + 1]; + for (Tuple shard : status) { + if (shard.v2().state().failed() == false) { + final int id = shard.v1().getId(); + assert gens[id] == null; + gens[id] = shard.v2().generation(); + } + } + return Arrays.asList(gens); + }))); + } + + public List indices() { + return List.copyOf(shardGenerations.keySet()); + } + /** * Computes the obsolete shard index generations that can be deleted if this instance was written to the repository. * @@ -91,14 +126,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - ShardGenerations updatedGenerations(final Map shardGenerations) { + ShardGenerations updatedGenerations(ShardGenerations updates) { final Map> updatedGenerations = new HashMap<>(this.shardGenerations); - shardGenerations.forEach(((indexId, updatedGens) -> { - final List existing = updatedGenerations.put(indexId, Arrays.asList(updatedGens)); + updates.shardGenerations.forEach(((indexId, updatedGens) -> { + final List existing = updatedGenerations.put(indexId, updatedGens); if (existing != null) { - for (int i = 0; i < updatedGens.length; ++i) { - if (updatedGens[i] == null) { - updatedGens[i] = existing.get(i); + for (int i = 0; i < updatedGens.size(); ++i) { + if (updatedGens.get(i) == null) { + updatedGens.set(i, existing.get(i)); } } } diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 1237419008679..5a309ab2ffc7f 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -88,6 +88,7 @@ import org.elasticsearch.repositories.RepositoryData; import org.elasticsearch.repositories.RepositoryException; import org.elasticsearch.repositories.RepositoryVerificationException; +import org.elasticsearch.repositories.ShardGenerations; import org.elasticsearch.snapshots.SnapshotCreationException; import org.elasticsearch.snapshots.SnapshotException; import org.elasticsearch.snapshots.SnapshotId; @@ -532,12 +533,13 @@ private void doDeleteShardSnapshots(SnapshotId snapshotId, long repositoryStateI updatedShardGenerations.computeIfAbsent(newGen.indexId, i -> new ArrayList<>()).add(newGen); } assert assertShardUpdatesCorrectlyOrdered(updatedShardGenerations.values()); - final RepositoryData newRepoData = repositoryData.removeSnapshot(snapshotId, updatedShardGenerations.entrySet().stream() + final RepositoryData newRepoData = repositoryData.removeSnapshot(snapshotId, new ShardGenerations( + updatedShardGenerations.entrySet().stream() .collect(Collectors.toMap( Map.Entry::getKey, entry -> entry.getValue().stream() .map(shardSnapshotMetaDeleteResult -> shardSnapshotMetaDeleteResult.newGeneration) - .toArray(String[]::new)))); + .collect(Collectors.toList()))))); // Write out new RepositoryData writeIndexGen(newRepoData, repositoryStateId, version); @@ -762,7 +764,7 @@ private void deleteIndexMetaDataBlobIgnoringErrors(SnapshotId snapshotId, IndexI @Override public SnapshotInfo finalizeSnapshot(final SnapshotId snapshotId, - final Map shardGenerations, + final ShardGenerations shardGenerations, final long startTime, final String failure, final int totalShards, @@ -772,8 +774,9 @@ public SnapshotInfo finalizeSnapshot(final SnapshotId snapshotId, final MetaData clusterMetaData, final Map userMetadata, final Version version) { + final List indices = shardGenerations.indices(); SnapshotInfo blobStoreSnapshot = new SnapshotInfo(snapshotId, - shardGenerations.keySet().stream().map(IndexId::getName).collect(Collectors.toList()), + indices.stream().map(IndexId::getName).collect(Collectors.toList()), startTime, failure, threadPool.absoluteTimeInMillis(), totalShards, shardFailures, includeGlobalState, userMetadata); @@ -788,7 +791,7 @@ public SnapshotInfo finalizeSnapshot(final SnapshotId snapshotId, globalMetaDataFormat.write(clusterMetaData, blobContainer(), snapshotId.getUUID(), false); // write the index metadata for each index in the snapshot - for (IndexId index : shardGenerations.keySet()) { + for (IndexId index : indices) { indexMetaDataFormat.write(clusterMetaData.index(index.getName()), indexContainer(index), snapshotId.getUUID(), false); } } catch (IOException ex) { diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 97dff76883256..c76a980aa8510 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -71,6 +71,7 @@ import org.elasticsearch.repositories.RepositoryData; import org.elasticsearch.repositories.RepositoryException; import org.elasticsearch.repositories.RepositoryMissingException; +import org.elasticsearch.repositories.ShardGenerations; import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; @@ -567,7 +568,7 @@ private void cleanupAfterError(Exception exception) { try { repositoriesService.repository(snapshot.snapshot().getRepository()) .finalizeSnapshot(snapshot.snapshot().getSnapshotId(), - shardGenerations(snapshot), + ShardGenerations.fromSnapshot(snapshot), snapshot.startTime(), ExceptionsHelper.detailedMessage(exception), 0, @@ -588,28 +589,6 @@ private void cleanupAfterError(Exception exception) { } } - private static Map shardGenerations(SnapshotsInProgress.Entry snapshot) { - final Map indexLookup = new HashMap<>(); - snapshot.indices().forEach(idx -> indexLookup.put(idx.getName(), idx)); - final Map>> res = new HashMap<>(); - for (ObjectObjectCursor shard : snapshot.shards()) { - final IndexId indexId = indexLookup.get(shard.key.getIndexName()); - assert indexId != null; - res.computeIfAbsent(indexId, k -> new ArrayList<>()).add(new Tuple<>(shard.key, shard.value)); - } - return res.entrySet().stream().collect( - Collectors.toMap(Map.Entry::getKey, entry -> { - final String[] gens = new String[entry.getValue().stream().mapToInt(s -> s.v1().getId()).max().orElse(0) + 1]; - for (Tuple shard : entry.getValue()) { - if (shard.v2().state().failed() == false) { - assert gens[shard.v1().getId()] == null; - gens[shard.v1().getId()] = shard.v2().generation(); - } - } - return gens; - })); - } - private static MetaData metaDataForSnapshot(SnapshotsInProgress.Entry snapshot, MetaData metaData) { if (snapshot.includeGlobalState() == false) { // Remove global state from the cluster state @@ -1032,7 +1011,7 @@ protected void doRun() { List shardFailures = extractFailure(entry.shards()); SnapshotInfo snapshotInfo = repository.finalizeSnapshot( snapshot.getSnapshotId(), - shardGenerations(entry), + ShardGenerations.fromSnapshot(entry), entry.startTime(), failure, entry.shards().size(), diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java index 1de6eccab3a5d..ddca7f50ad0e5 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java @@ -159,7 +159,7 @@ public void initializeSnapshot(SnapshotId snapshotId, List indices, Met } @Override - public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, Map indices, long startTime, String failure, + public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, ShardGenerations indices, long startTime, String failure, int totalShards, List shardFailures, long repositoryStateId, boolean includeGlobalState, MetaData metaData, Map userMetadata, Version version) { diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java index 2a0afecd5e2b1..3f4aad38c6b88 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java @@ -21,7 +21,6 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContent; @@ -77,21 +76,21 @@ public void testAddSnapshots() { Map indexIdMap = repositoryData.getIndices(); // test that adding a snapshot and its indices works SnapshotId newSnapshot = new SnapshotId(randomAlphaOfLength(7), UUIDs.randomBase64UUID()); - Map indices = new HashMap<>(); + Map> indices = new HashMap<>(); Set newIndices = new HashSet<>(); int numNew = randomIntBetween(1, 10); for (int i = 0; i < numNew; i++) { IndexId indexId = new IndexId(randomAlphaOfLength(7), UUIDs.randomBase64UUID()); newIndices.add(indexId); - indices.put(indexId, Strings.EMPTY_ARRAY); + indices.put(indexId, Collections.emptyList()); } int numOld = randomIntBetween(1, indexIdMap.size()); List indexNames = new ArrayList<>(indexIdMap.keySet()); for (int i = 0; i < numOld; i++) { - indices.put(indexIdMap.get(indexNames.get(i)), new String[]{"1"}); + indices.put(indexIdMap.get(indexNames.get(i)), Collections.singletonList("1")); } RepositoryData newRepoData = repositoryData.addSnapshot(newSnapshot, - randomFrom(SnapshotState.SUCCESS, SnapshotState.PARTIAL, SnapshotState.FAILED), indices); + randomFrom(SnapshotState.SUCCESS, SnapshotState.PARTIAL, SnapshotState.FAILED), new ShardGenerations(indices)); // verify that the new repository data has the new snapshot and its indices assertTrue(newRepoData.getSnapshotIds().contains(newSnapshot)); for (IndexId indexId : indices.keySet()) { @@ -138,7 +137,7 @@ public void testRemoveSnapshot() { List snapshotIds = new ArrayList<>(repositoryData.getSnapshotIds()); assertThat(snapshotIds.size(), greaterThan(0)); SnapshotId removedSnapshotId = snapshotIds.remove(randomIntBetween(0, snapshotIds.size() - 1)); - RepositoryData newRepositoryData = repositoryData.removeSnapshot(removedSnapshotId, Collections.emptyMap()); + RepositoryData newRepositoryData = repositoryData.removeSnapshot(removedSnapshotId, ShardGenerations.EMPTY); // make sure the repository data's indices no longer contain the removed snapshot for (final IndexId indexId : newRepositoryData.getIndices().values()) { assertFalse(newRepositoryData.getSnapshots(indexId).contains(removedSnapshotId)); @@ -161,7 +160,7 @@ public void testResolveIndexId() { public void testGetSnapshotState() { final SnapshotId snapshotId = new SnapshotId(randomAlphaOfLength(8), UUIDs.randomBase64UUID()); final SnapshotState state = randomFrom(SnapshotState.values()); - final RepositoryData repositoryData = RepositoryData.EMPTY.addSnapshot(snapshotId, state, Collections.emptyMap()); + final RepositoryData repositoryData = RepositoryData.EMPTY.addSnapshot(snapshotId, state, ShardGenerations.EMPTY); assertEquals(state, repositoryData.getSnapshotState(snapshotId)); assertNull(repositoryData.getSnapshotState(new SnapshotId(randomAlphaOfLength(8), UUIDs.randomBase64UUID()))); } @@ -262,11 +261,12 @@ public static RepositoryData generateRandomRepoData() { for (int i = 0; i < numSnapshots; i++) { final SnapshotId snapshotId = new SnapshotId(randomAlphaOfLength(8), UUIDs.randomBase64UUID()); final List someIndices = indices.subList(0, randomIntBetween(1, numIndices)); - final Map shardGenerations = new HashMap<>(); + final Map> shardGenerations = new HashMap<>(); for (IndexId someIndex : someIndices) { - shardGenerations.put(someIndex, new String[]{"1"}); + shardGenerations.put(someIndex, Collections.singletonList("1")); } - repositoryData = repositoryData.addSnapshot(snapshotId, randomFrom(SnapshotState.values()), shardGenerations); + repositoryData = + repositoryData.addSnapshot(snapshotId, randomFrom(SnapshotState.values()), new ShardGenerations(shardGenerations)); } return repositoryData; } diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java index d6dbfbda83560..ee9033d47bd4a 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java @@ -43,6 +43,7 @@ import org.elasticsearch.index.store.StoreFileMetaData; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.ShardGenerations; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.snapshots.SnapshotId; @@ -165,7 +166,8 @@ public void testSnapshotWithConflictingName() throws IOException { final String shardGen = snapshotShard(shard, snapshot, repository); final Snapshot snapshotWithSameName = new Snapshot(repository.getMetadata().name(), new SnapshotId( snapshot.getSnapshotId().getName(), "_uuid2")); - repository.finalizeSnapshot(snapshot.getSnapshotId(), Collections.singletonMap(indexId, new String[] {shardGen}), + repository.finalizeSnapshot(snapshot.getSnapshotId(), + new ShardGenerations(Collections.singletonMap(indexId, Collections.singletonList(shardGen))), 0L, null, 1, Collections.emptyList(), -1L, false, MetaData.builder().put(shard.indexSettings().getIndexMetaData(), false).build(), Collections.emptyMap(), Version.CURRENT); IndexShardSnapshotFailedException isfe = expectThrows(IndexShardSnapshotFailedException.class, diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java index 50786b3928b0d..abab0a469128e 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; @@ -36,6 +35,7 @@ import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryData; import org.elasticsearch.repositories.RepositoryException; +import org.elasticsearch.repositories.ShardGenerations; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotState; @@ -180,7 +180,7 @@ public void testIndexGenerationalFiles() throws Exception { // removing a snapshot and writing to a new index generational file repositoryData = repository.getRepositoryData().removeSnapshot( - repositoryData.getSnapshotIds().iterator().next(), Collections.emptyMap()); + repositoryData.getSnapshotIds().iterator().next(), ShardGenerations.EMPTY); repository.writeIndexGen(repositoryData, repositoryData.getGenId(), Version.CURRENT); assertEquals(repository.getRepositoryData(), repositoryData); assertThat(repository.latestIndexBlobId(), equalTo(2L)); @@ -245,12 +245,12 @@ private RepositoryData addRandomSnapshotsToRepoData(RepositoryData repoData, boo for (int i = 0; i < numSnapshots; i++) { SnapshotId snapshotId = new SnapshotId(randomAlphaOfLength(8), UUIDs.randomBase64UUID()); int numIndices = inclIndices ? randomIntBetween(0, 20) : 0; - Map indexIds = new HashMap<>(); + Map> indexIds = new HashMap<>(); for (int j = 0; j < numIndices; j++) { - indexIds.put(new IndexId(randomAlphaOfLength(8), UUIDs.randomBase64UUID()), Strings.EMPTY_ARRAY); + indexIds.put(new IndexId(randomAlphaOfLength(8), UUIDs.randomBase64UUID()), Collections.emptyList()); } repoData = repoData.addSnapshot(snapshotId, - randomFrom(SnapshotState.SUCCESS, SnapshotState.PARTIAL, SnapshotState.FAILED), indexIds); + randomFrom(SnapshotState.SUCCESS, SnapshotState.PARTIAL, SnapshotState.FAILED), new ShardGenerations(indexIds)); } return repoData; } diff --git a/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockEventuallyConsistentRepositoryTests.java b/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockEventuallyConsistentRepositoryTests.java index 34476ebf8dfe6..a9a82d7b517f7 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockEventuallyConsistentRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockEventuallyConsistentRepositoryTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.repositories.ShardGenerations; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.test.ESTestCase; @@ -159,19 +160,19 @@ public void testOverwriteSnapshotInfoBlob() { // We create a snap- blob for snapshot "foo" in the first generation final SnapshotId snapshotId = new SnapshotId("foo", UUIDs.randomBase64UUID()); - repository.finalizeSnapshot(snapshotId, Collections.emptyMap(), 1L, null, 5, Collections.emptyList(), + repository.finalizeSnapshot(snapshotId, ShardGenerations.EMPTY, 1L, null, 5, Collections.emptyList(), -1L, false, MetaData.EMPTY_META_DATA, Collections.emptyMap(), Version.CURRENT); // We try to write another snap- blob for "foo" in the next generation. It fails because the content differs. final AssertionError assertionError = expectThrows(AssertionError.class, () -> repository.finalizeSnapshot( - snapshotId, Collections.emptyMap(), 1L, null, 6, Collections.emptyList(), + snapshotId, ShardGenerations.EMPTY, 1L, null, 6, Collections.emptyList(), 0, false, MetaData.EMPTY_META_DATA, Collections.emptyMap(), Version.CURRENT)); assertThat(assertionError.getMessage(), equalTo("\nExpected: <6>\n but: was <5>")); // We try to write yet another snap- blob for "foo" in the next generation. // It passes cleanly because the content of the blob except for the timestamps. - repository.finalizeSnapshot(snapshotId, Collections.emptyMap(), 1L, null, 5, Collections.emptyList(), + repository.finalizeSnapshot(snapshotId, ShardGenerations.EMPTY, 1L, null, 5, Collections.emptyList(), 0, false, MetaData.EMPTY_META_DATA, Collections.emptyMap(), Version.CURRENT); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java index 5ff940f4f8715..8b6328ca18f18 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java @@ -32,6 +32,7 @@ import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryData; +import org.elasticsearch.repositories.ShardGenerations; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.snapshots.SnapshotShardFailure; @@ -100,7 +101,7 @@ public void initializeSnapshot(SnapshotId snapshotId, List indices, Met } @Override - public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, Map shardGenerations, long startTime, String failure, + public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenerations, long startTime, String failure, int totalShards, List shardFailures, long repositoryStateId, boolean includeGlobalState, MetaData metaData, Map userMetadata, Version version) { diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index c28e2a25d72d2..7c2b14ecbb038 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -59,6 +59,7 @@ import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryData; +import org.elasticsearch.repositories.ShardGenerations; import org.elasticsearch.repositories.blobstore.FileRestoreContext; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; @@ -256,7 +257,7 @@ public void initializeSnapshot(SnapshotId snapshotId, List indices, Met } @Override - public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, Map shardGenerations, long startTime, String failure, + public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenerations, long startTime, String failure, int totalShards, List shardFailures, long repositoryStateId, boolean includeGlobalState, MetaData metaData, Map userMetadata, Version version) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java index 575ca5950425c..f7b870ccbfb13 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java @@ -37,6 +37,7 @@ import org.elasticsearch.repositories.FilterRepository; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.ShardGenerations; import java.io.Closeable; import java.io.IOException; @@ -91,7 +92,7 @@ public void initializeSnapshot(SnapshotId snapshotId, List indices, Met } @Override - public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, Map shardGenerations, long startTime, String failure, + public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenerations, long startTime, String failure, int totalShards, List shardFailures, long repositoryStateId, boolean includeGlobalState, MetaData metaData, Map userMetadata, Version version) { @@ -100,7 +101,7 @@ public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, Map(shardGenerations.keySet()), metaData), userMetadata, version); + includeGlobalState, metadataToSnapshot(shardGenerations.indices(), metaData), userMetadata, version); } catch (IOException ex) { throw new UncheckedIOException(ex); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java index 09c61aaa819f0..e0da694cee580 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java @@ -59,6 +59,7 @@ import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.ShardGenerations; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.threadpool.ThreadPool; import org.hamcrest.Matchers; @@ -207,7 +208,9 @@ public void testRestoreMinmal() throws IOException { indexShardSnapshotStatus, Version.CURRENT, future); future.actionGet(); repository.finalizeSnapshot( - snapshotId, Collections.singletonMap(indexId, new String[]{indexShardSnapshotStatus.generation()}), + snapshotId, + new ShardGenerations( + Collections.singletonMap(indexId, Collections.singletonList(indexShardSnapshotStatus.generation()))), indexShardSnapshotStatus.asCopy().getStartTime(), null, 1, Collections.emptyList(), repository.getRepositoryData().getGenId(), true, MetaData.builder().put(shard.indexSettings().getIndexMetaData(), false).build(), Collections.emptyMap(),