From 37dbb051149e4ad4ccf5c66dbf7ffb796284f300 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Thu, 23 Apr 2020 12:41:07 +0200 Subject: [PATCH] Allow Deleting Multiple Snapshots at Once (#55474) Adds deleting multiple snapshots in one go without significantly changing the mechanics of snapshot deletes otherwise. This change does not yet allow mixing snapshot delete and abort. Abort is still only allowed for a single snapshot delete by exact name. --- .../client/SnapshotRequestConverters.java | 2 +- .../SnapshotRequestConvertersTests.java | 2 +- .../SnapshotClientDocumentationIT.java | 2 +- .../snapshot-restore/take-snapshot.asciidoc | 9 + .../repositories/s3/S3Repository.java | 7 +- .../delete/DeleteSnapshotRequest.java | 50 +++--- .../delete/DeleteSnapshotRequestBuilder.java | 10 +- .../delete/TransportDeleteSnapshotAction.java | 3 +- .../client/ClusterAdminClient.java | 2 +- .../org/elasticsearch/client/Requests.java | 8 +- .../client/support/AbstractClient.java | 4 +- .../cluster/SnapshotDeletionsInProgress.java | 54 ++++-- .../repositories/FilterRepository.java | 7 +- .../repositories/RepositoriesService.java | 2 +- .../repositories/Repository.java | 9 +- .../repositories/RepositoryData.java | 56 +++--- .../blobstore/BlobStoreRepository.java | 148 ++++++++-------- .../repositories/blobstore/package-info.java | 2 +- .../cluster/RestDeleteSnapshotAction.java | 4 +- .../snapshots/RestoreService.java | 4 +- .../snapshots/SnapshotsService.java | 159 +++++++++++------- .../elasticsearch/snapshots/package-info.java | 2 +- .../ClusterSerializationTests.java | 2 +- .../RepositoriesServiceTests.java | 5 +- .../repositories/RepositoryDataTests.java | 5 +- .../blobstore/BlobStoreRepositoryTests.java | 4 +- .../SharedClusterSnapshotRestoreIT.java | 45 ++++- .../index/shard/RestoreOnlyRepository.java | 5 +- .../xpack/ccr/repository/CcrRepository.java | 5 +- .../core/ilm/CleanupSnapshotStepTests.java | 4 +- .../xpack/slm/SnapshotRetentionTask.java | 2 + .../xpack/slm/SnapshotRetentionTaskTests.java | 3 +- 32 files changed, 379 insertions(+), 247 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java index 3d033bc2890a3..796845d352528 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java @@ -176,7 +176,7 @@ static Request restoreSnapshot(RestoreSnapshotRequest restoreSnapshotRequest) th static Request deleteSnapshot(DeleteSnapshotRequest deleteSnapshotRequest) { String endpoint = new RequestConverters.EndpointBuilder().addPathPartAsIs("_snapshot") .addPathPart(deleteSnapshotRequest.repository()) - .addPathPart(deleteSnapshotRequest.snapshot()) + .addCommaSeparatedPathParts(deleteSnapshotRequest.snapshots()) .build(); Request request = new Request(HttpDelete.METHOD_NAME, endpoint); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotRequestConvertersTests.java index 668fa8fe6b1b2..0379150c6cd4b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotRequestConvertersTests.java @@ -269,7 +269,7 @@ public void testDeleteSnapshot() { DeleteSnapshotRequest deleteSnapshotRequest = new DeleteSnapshotRequest(); deleteSnapshotRequest.repository(repository); - deleteSnapshotRequest.snapshot(snapshot); + deleteSnapshotRequest.snapshots(snapshot); RequestConvertersTests.setRandomMasterTimeout(deleteSnapshotRequest, expectedParams); Request request = SnapshotRequestConverters.deleteSnapshot(deleteSnapshotRequest); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java index 33b5480d2d5c1..d93b5a284caf8 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java @@ -753,7 +753,7 @@ public void testSnapshotDeleteSnapshot() throws IOException { // tag::delete-snapshot-request DeleteSnapshotRequest request = new DeleteSnapshotRequest(repositoryName); - request.snapshot(snapshotName); + request.snapshots(snapshotName); // end::delete-snapshot-request // tag::delete-snapshot-request-masterTimeout diff --git a/docs/reference/snapshot-restore/take-snapshot.asciidoc b/docs/reference/snapshot-restore/take-snapshot.asciidoc index 20c2b4837306b..aeeb18ecfc2d6 100644 --- a/docs/reference/snapshot-restore/take-snapshot.asciidoc +++ b/docs/reference/snapshot-restore/take-snapshot.asciidoc @@ -193,6 +193,15 @@ created the snapshotting process will be aborted and all files created as part o cleaned. Therefore, the delete snapshot operation can be used to cancel long running snapshot operations that were started by mistake. +It is also possible to delete multiple snapshots from a repository in one go, for example: + +[source,console] +----------------------------------- +DELETE /_snapshot/my_backup/my_backup,my_fs_backup +DELETE /_snapshot/my_backup/snap* +----------------------------------- +// TEST[skip:no my_fs_backup] + A repository can be unregistered using the following command: [source,console] diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index 0d8e620c89219..9fdaec59a342c 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -52,6 +52,7 @@ import org.elasticsearch.threadpool.Scheduler; import org.elasticsearch.threadpool.ThreadPool; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -286,12 +287,12 @@ public void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenera } @Override - public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, Version repositoryMetaVersion, - ActionListener listener) { + public void deleteSnapshots(Collection snapshotIds, long repositoryStateId, Version repositoryMetaVersion, + ActionListener listener) { if (SnapshotsService.useShardGenerations(repositoryMetaVersion) == false) { listener = delayedListener(listener); } - super.deleteSnapshot(snapshotId, repositoryStateId, repositoryMetaVersion, listener); + super.deleteSnapshots(snapshotIds, repositoryStateId, repositoryMetaVersion, listener); } /** diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java index 93581c937c50e..6089e7ab15187 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.snapshots.SnapshotsService; import java.io.IOException; @@ -31,15 +32,14 @@ /** * Delete snapshot request *

- * Delete snapshot request removes the snapshot record from the repository and cleans up all - * files that are associated with this particular snapshot. All files that are shared with - * at least one other existing snapshot are left intact. + * Delete snapshot request removes snapshots from the repository and cleans up all files that are associated with the snapshots. + * All files that are shared with at least one other existing snapshot are left intact. */ public class DeleteSnapshotRequest extends MasterNodeRequest { private String repository; - private String snapshot; + private String[] snapshots; /** * Constructs a new delete snapshots request @@ -48,14 +48,14 @@ public DeleteSnapshotRequest() { } /** - * Constructs a new delete snapshots request with repository and snapshot name + * Constructs a new delete snapshots request with repository and snapshot names * * @param repository repository name - * @param snapshot snapshot name + * @param snapshots snapshot names */ - public DeleteSnapshotRequest(String repository, String snapshot) { + public DeleteSnapshotRequest(String repository, String... snapshots) { this.repository = repository; - this.snapshot = snapshot; + this.snapshots = snapshots; } /** @@ -70,14 +70,26 @@ public DeleteSnapshotRequest(String repository) { public DeleteSnapshotRequest(StreamInput in) throws IOException { super(in); repository = in.readString(); - snapshot = in.readString(); + if (in.getVersion().onOrAfter(SnapshotsService.MULTI_DELETE_VERSION)) { + snapshots = in.readStringArray(); + } else { + snapshots = new String[] {in.readString()}; + } } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(repository); - out.writeString(snapshot); + if (out.getVersion().onOrAfter(SnapshotsService.MULTI_DELETE_VERSION)) { + out.writeStringArray(snapshots); + } else { + if (snapshots.length != 1) { + throw new IllegalArgumentException( + "Can't write snapshot delete with more than one snapshot to version [" + out.getVersion() + "]"); + } + out.writeString(snapshots[0]); + } } @Override @@ -86,8 +98,8 @@ public ActionRequestValidationException validate() { if (repository == null) { validationException = addValidationError("repository is missing", validationException); } - if (snapshot == null) { - validationException = addValidationError("snapshot is missing", validationException); + if (snapshots == null || snapshots.length == 0) { + validationException = addValidationError("snapshots are missing", validationException); } return validationException; } @@ -108,21 +120,21 @@ public String repository() { } /** - * Returns repository name + * Returns snapshot names * - * @return repository name + * @return snapshot names */ - public String snapshot() { - return this.snapshot; + public String[] snapshots() { + return this.snapshots; } /** - * Sets snapshot name + * Sets snapshot names * * @return this request */ - public DeleteSnapshotRequest snapshot(String snapshot) { - this.snapshot = snapshot; + public DeleteSnapshotRequest snapshots(String... snapshots) { + this.snapshots = snapshots; return this; } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequestBuilder.java index 1e47160903c85..ac429d85269e4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequestBuilder.java @@ -39,8 +39,8 @@ public DeleteSnapshotRequestBuilder(ElasticsearchClient client, DeleteSnapshotAc /** * Constructs delete snapshot request builder with specified repository and snapshot names */ - public DeleteSnapshotRequestBuilder(ElasticsearchClient client, DeleteSnapshotAction action, String repository, String snapshot) { - super(client, action, new DeleteSnapshotRequest(repository, snapshot)); + public DeleteSnapshotRequestBuilder(ElasticsearchClient client, DeleteSnapshotAction action, String repository, String... snapshots) { + super(client, action, new DeleteSnapshotRequest(repository, snapshots)); } /** @@ -57,11 +57,11 @@ public DeleteSnapshotRequestBuilder setRepository(String repository) { /** * Sets the snapshot name * - * @param snapshot snapshot name + * @param snapshots snapshot names * @return this builder */ - public DeleteSnapshotRequestBuilder setSnapshot(String snapshot) { - request.snapshot(snapshot); + public DeleteSnapshotRequestBuilder setSnapshots(String... snapshots) { + request.snapshots(snapshots); return this; } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java index 556237d447068..912b492015a69 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java @@ -35,6 +35,7 @@ import org.elasticsearch.transport.TransportService; import java.io.IOException; +import java.util.Arrays; /** * Transport action for delete snapshot operation @@ -70,7 +71,7 @@ protected ClusterBlockException checkBlock(DeleteSnapshotRequest request, Cluste @Override protected void masterOperation(final DeleteSnapshotRequest request, ClusterState state, final ActionListener listener) { - snapshotsService.deleteSnapshot(request.repository(), request.snapshot(), + snapshotsService.deleteSnapshots(request.repository(), Arrays.asList(request.snapshots()), ActionListener.map(listener, v -> new AcknowledgedResponse(true))); } } diff --git a/server/src/main/java/org/elasticsearch/client/ClusterAdminClient.java b/server/src/main/java/org/elasticsearch/client/ClusterAdminClient.java index 2321c6b5f7d44..19e876eebcd36 100644 --- a/server/src/main/java/org/elasticsearch/client/ClusterAdminClient.java +++ b/server/src/main/java/org/elasticsearch/client/ClusterAdminClient.java @@ -529,7 +529,7 @@ public interface ClusterAdminClient extends ElasticsearchClient { /** * Delete snapshot. */ - DeleteSnapshotRequestBuilder prepareDeleteSnapshot(String repository, String snapshot); + DeleteSnapshotRequestBuilder prepareDeleteSnapshot(String repository, String... snapshot); /** * Restores a snapshot. diff --git a/server/src/main/java/org/elasticsearch/client/Requests.java b/server/src/main/java/org/elasticsearch/client/Requests.java index d9b4794dc461a..fb9fd9ba15933 100644 --- a/server/src/main/java/org/elasticsearch/client/Requests.java +++ b/server/src/main/java/org/elasticsearch/client/Requests.java @@ -526,14 +526,14 @@ public static RestoreSnapshotRequest restoreSnapshotRequest(String repository, S } /** - * Deletes a snapshot + * Deletes snapshots * - * @param snapshot snapshot name + * @param snapshots snapshot names * @param repository repository name * @return delete snapshot request */ - public static DeleteSnapshotRequest deleteSnapshotRequest(String repository, String snapshot) { - return new DeleteSnapshotRequest(repository, snapshot); + public static DeleteSnapshotRequest deleteSnapshotRequest(String repository, String... snapshots) { + return new DeleteSnapshotRequest(repository, snapshots); } /** diff --git a/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java index 04f3f16ec0fcb..5d6cd253f3343 100644 --- a/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java +++ b/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java @@ -976,8 +976,8 @@ public void deleteSnapshot(DeleteSnapshotRequest request, ActionListener snapshots; + private final String repoName; private final long startTime; private final long repositoryStateId; - public Entry(Snapshot snapshot, long startTime, long repositoryStateId) { - this.snapshot = snapshot; + public Entry(List snapshots, String repoName, long startTime, long repositoryStateId) { + this.snapshots = snapshots; + assert snapshots.size() == new HashSet<>(snapshots).size() : "Duplicate snapshot ids in " + snapshots; + this.repoName = repoName; this.startTime = startTime; this.repositoryStateId = repositoryStateId; assert repositoryStateId > RepositoryData.EMPTY_REPO_GEN : @@ -180,16 +190,20 @@ public Entry(Snapshot snapshot, long startTime, long repositoryStateId) { } public Entry(StreamInput in) throws IOException { - this.snapshot = new Snapshot(in); + if (in.getVersion().onOrAfter(SnapshotsService.MULTI_DELETE_VERSION)) { + this.repoName = in.readString(); + this.snapshots = in.readList(SnapshotId::new); + } else { + final Snapshot snapshot = new Snapshot(in); + this.snapshots = Collections.singletonList(snapshot.getSnapshotId()); + this.repoName = snapshot.getRepository(); + } this.startTime = in.readVLong(); this.repositoryStateId = in.readLong(); } - /** - * The snapshot to delete. - */ - public Snapshot getSnapshot() { - return snapshot; + public List getSnapshots() { + return snapshots; } /** @@ -208,26 +222,34 @@ public boolean equals(Object o) { return false; } Entry that = (Entry) o; - return snapshot.equals(that.snapshot) + return repoName.equals(that.repoName) + && snapshots.equals(that.snapshots) && startTime == that.startTime && repositoryStateId == that.repositoryStateId; } @Override public int hashCode() { - return Objects.hash(snapshot, startTime, repositoryStateId); + return Objects.hash(snapshots, repoName, startTime, repositoryStateId); } @Override public void writeTo(StreamOutput out) throws IOException { - snapshot.writeTo(out); + if (out.getVersion().onOrAfter(SnapshotsService.MULTI_DELETE_VERSION)) { + out.writeString(repoName); + out.writeCollection(snapshots); + } else { + assert snapshots.size() == 1 : "Only single deletion allowed in mixed version cluster containing [" + out.getVersion() + + "] but saw " + snapshots; + new Snapshot(repoName, snapshots.get(0)).writeTo(out); + } out.writeVLong(startTime); out.writeLong(repositoryStateId); } @Override public String repository() { - return snapshot.getRepository(); + return repoName; } @Override diff --git a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java index a418ac4991047..7d36931c1aec8 100644 --- a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java @@ -39,6 +39,7 @@ import org.elasticsearch.snapshots.SnapshotShardFailure; import java.io.IOException; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -92,9 +93,9 @@ public void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenera } @Override - public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, Version repositoryMetaVersion, - ActionListener listener) { - in.deleteSnapshot(snapshotId, repositoryStateId, repositoryMetaVersion, listener); + public void deleteSnapshots(Collection snapshotIds, long repositoryStateId, Version repositoryMetaVersion, + ActionListener listener) { + in.deleteSnapshots(snapshotIds, repositoryStateId, repositoryMetaVersion, listener); } @Override diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java index 1e342ac51c4d2..4bcf2f4dbe42a 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java @@ -487,7 +487,7 @@ private static boolean isRepositoryInUse(ClusterState clusterState, String repos SnapshotDeletionsInProgress deletionsInProgress = clusterState.custom(SnapshotDeletionsInProgress.TYPE); if (deletionsInProgress != null) { for (SnapshotDeletionsInProgress.Entry entry : deletionsInProgress.getEntries()) { - if (entry.getSnapshot().getRepository().equals(repository)) { + if (entry.repository().equals(repository)) { return true; } } diff --git a/server/src/main/java/org/elasticsearch/repositories/Repository.java b/server/src/main/java/org/elasticsearch/repositories/Repository.java index 15297d4f699b4..8212294cc6b58 100644 --- a/server/src/main/java/org/elasticsearch/repositories/Repository.java +++ b/server/src/main/java/org/elasticsearch/repositories/Repository.java @@ -40,6 +40,7 @@ import org.elasticsearch.snapshots.SnapshotShardFailure; import java.io.IOException; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -153,15 +154,15 @@ void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenerations, ActionListener> listener); /** - * Deletes snapshot + * Deletes snapshots * - * @param snapshotId snapshot id + * @param snapshotIds snapshot ids * @param repositoryStateId the unique id identifying the state of the repository when the snapshot deletion began * @param repositoryMetaVersion version of the updated repository metadata to write * @param listener completion listener */ - void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, Version repositoryMetaVersion, ActionListener listener); - + void deleteSnapshots(Collection snapshotIds, long repositoryStateId, Version repositoryMetaVersion, + ActionListener listener); /** * Returns snapshot throttle time in nanoseconds */ diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java index 487ec1dff0cce..be1123b2c06b8 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java @@ -179,14 +179,23 @@ public Map getIndices() { * Returns the list of {@link IndexId} that have their snapshots updated but not removed (because they are still referenced by other * snapshots) after removing the given snapshot from the repository. * - * @param snapshotId SnapshotId to remove + * @param snapshotIds SnapshotId to remove * @return List of indices that are changed but not removed */ - public List indicesToUpdateAfterRemovingSnapshot(SnapshotId snapshotId) { + public List indicesToUpdateAfterRemovingSnapshot(Collection snapshotIds) { return indexSnapshots.entrySet().stream() - .filter(entry -> entry.getValue().size() > 1 && entry.getValue().contains(snapshotId)) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); + .filter(entry -> { + final Collection existingIds = entry.getValue(); + if (snapshotIds.containsAll(existingIds)) { + return existingIds.size() > snapshotIds.size(); + } + for (SnapshotId snapshotId : snapshotIds) { + if (entry.getValue().contains(snapshotId)) { + return true; + } + } + return false; + }).map(Map.Entry::getKey).collect(Collectors.toList()); } /** @@ -244,43 +253,40 @@ public RepositoryData withGenId(long newGeneration) { } /** - * Remove a snapshot and remove any indices that no longer exist in the repository due to the deletion of the snapshot. + * Remove snapshots and remove any indices that no longer exist in the repository due to the deletion of the snapshots. * - * @param snapshotId Snapshot Id + * @param snapshots Snapshot ids to remove * @param updatedShardGenerations Shard generations that changed as a result of removing the snapshot. * The {@code String[]} passed for each {@link IndexId} contains the new shard generation id for each * changed shard indexed by its shardId */ - public RepositoryData removeSnapshot(final SnapshotId snapshotId, final ShardGenerations updatedShardGenerations) { - Map newSnapshotIds = snapshotIds.values().stream() - .filter(id -> !snapshotId.equals(id)) + public RepositoryData removeSnapshots(final Collection snapshots, final ShardGenerations updatedShardGenerations) { + Map newSnapshotIds = snapshotIds.values().stream().filter(sn -> snapshots.contains(sn) == false) .collect(Collectors.toMap(SnapshotId::getUUID, Function.identity())); - if (newSnapshotIds.size() == snapshotIds.size()) { - throw new ResourceNotFoundException("Attempting to remove non-existent snapshot [{}] from repository data", snapshotId); + if (newSnapshotIds.size() != snapshotIds.size() - snapshots.size()) { + final Collection notFound = new HashSet<>(snapshots); + notFound.removeAll(snapshotIds.values()); + throw new ResourceNotFoundException("Attempting to remove non-existent snapshots {} from repository data", notFound); } Map newSnapshotStates = new HashMap<>(snapshotStates); - newSnapshotStates.remove(snapshotId.getUUID()); final Map newSnapshotVersions = new HashMap<>(snapshotVersions); - newSnapshotVersions.remove(snapshotId.getUUID()); + for (SnapshotId snapshotId : snapshots) { + newSnapshotStates.remove(snapshotId.getUUID()); + newSnapshotVersions.remove(snapshotId.getUUID()); + } Map> indexSnapshots = new HashMap<>(); for (final IndexId indexId : indices.values()) { - List remaining; List snapshotIds = this.indexSnapshots.get(indexId); assert snapshotIds != null; - final int listIndex = snapshotIds.indexOf(snapshotId); - if (listIndex > -1) { - if (snapshotIds.size() == 1) { - // removing the snapshot will mean no more snapshots - // have this index, so just skip over it - continue; - } - remaining = new ArrayList<>(snapshotIds); - remaining.remove(listIndex); + List remaining = new ArrayList<>(snapshotIds); + if (remaining.removeAll(snapshots)) { remaining = Collections.unmodifiableList(remaining); } else { remaining = snapshotIds; } - indexSnapshots.put(indexId, remaining); + if (remaining.isEmpty() == false) { + indexSnapshots.put(indexId, remaining); + } } return new RepositoryData(genId, newSnapshotIds, newSnapshotStates, newSnapshotVersions, indexSnapshots, diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index adaddfe0fda0a..71b63b6e74200 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -83,11 +83,9 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.Index; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.snapshots.IndexShardRestoreFailedException; -import org.elasticsearch.index.snapshots.IndexShardSnapshotException; import org.elasticsearch.index.snapshots.IndexShardSnapshotFailedException; import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot; @@ -108,8 +106,6 @@ import org.elasticsearch.repositories.RepositoryVerificationException; import org.elasticsearch.snapshots.SnapshotCreationException; import org.elasticsearch.repositories.ShardGenerations; -import org.elasticsearch.snapshots.ConcurrentSnapshotExecutionException; -import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.snapshots.SnapshotException; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; @@ -512,28 +508,21 @@ public void initializeSnapshot(SnapshotId snapshotId, List indices, Met } @Override - public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, Version repositoryMetaVersion, - ActionListener listener) { + public void deleteSnapshots(Collection snapshotIds, long repositoryStateId, Version repositoryMetaVersion, + ActionListener listener) { if (isReadOnly()) { listener.onFailure(new RepositoryException(metadata.name(), "cannot delete snapshot from a readonly repository")); } else { - final long latestKnownGen = latestKnownRepoGen.get(); - if (latestKnownGen > repositoryStateId) { - listener.onFailure(new ConcurrentSnapshotExecutionException( - new Snapshot(metadata.name(), snapshotId), "Another concurrent operation moved repo generation to [ " + latestKnownGen - + "] but this delete assumed generation [" + repositoryStateId + "]")); - return; - } try { final Map rootBlobs = blobContainer().listBlobs(); final RepositoryData repositoryData = safeRepositoryData(repositoryStateId, rootBlobs); // Cache the indices that were found before writing out the new index-N blob so that a stuck master will never // delete an index that was created by another master node after writing this index-N blob. final Map foundIndices = blobStore().blobContainer(indicesPath()).children(); - doDeleteShardSnapshots(snapshotId, repositoryStateId, foundIndices, rootBlobs, repositoryData, + doDeleteShardSnapshots(snapshotIds, repositoryStateId, foundIndices, rootBlobs, repositoryData, SnapshotsService.useShardGenerations(repositoryMetaVersion), listener); } catch (Exception ex) { - listener.onFailure(new RepositoryException(metadata.name(), "failed to delete snapshot [" + snapshotId + "]", ex)); + listener.onFailure(new RepositoryException(metadata.name(), "failed to delete snapshots " + snapshotIds, ex)); } } } @@ -577,7 +566,7 @@ private RepositoryData safeRepositoryData(long repositoryStateId, Map foundIndices, + private void doDeleteShardSnapshots(Collection snapshotIds, long repositoryStateId, Map foundIndices, Map rootBlobs, RepositoryData repositoryData, boolean writeShardGens, ActionListener listener) { if (writeShardGens) { // First write the new shard state metadata (with the removed snapshot) and compute deletion targets - final StepListener> writeShardMetadataAndComputeDeletesStep = new StepListener<>(); - writeUpdatedShardMetadataAndComputeDeletes(snapshotId, repositoryData, true, writeShardMetadataAndComputeDeletesStep); + final StepListener> writeShardMetaDataAndComputeDeletesStep = new StepListener<>(); + writeUpdatedShardMetaDataAndComputeDeletes(snapshotIds, repositoryData, true, writeShardMetaDataAndComputeDeletesStep); // Once we have put the new shard-level metadata into place, we can update the repository metadata as follows: - // 1. Remove the snapshot from the list of existing snapshots + // 1. Remove the snapshots from the list of existing snapshots // 2. Update the index shard generations of all updated shard folders // // Note: If we fail updating any of the individual shard paths, none of them are changed since the newly created // index-${gen_uuid} will not be referenced by the existing RepositoryData and new RepositoryData is only // written if all shard paths have been successfully updated. final StepListener writeUpdatedRepoDataStep = new StepListener<>(); - writeShardMetadataAndComputeDeletesStep.whenComplete(deleteResults -> { + writeShardMetaDataAndComputeDeletesStep.whenComplete(deleteResults -> { final ShardGenerations.Builder builder = ShardGenerations.builder(); for (ShardSnapshotMetaDeleteResult newGen : deleteResults) { builder.put(newGen.indexId, newGen.shardId, newGen.newGeneration); } - final RepositoryData updatedRepoData = repositoryData.removeSnapshot(snapshotId, builder.build()); + final RepositoryData updatedRepoData = repositoryData.removeSnapshots(snapshotIds, builder.build()); writeIndexGen(updatedRepoData, repositoryStateId, true, Function.identity(), ActionListener.wrap(v -> writeUpdatedRepoDataStep.onResponse(updatedRepoData), listener::onFailure)); }, listener::onFailure); @@ -617,20 +606,20 @@ private void doDeleteShardSnapshots(SnapshotId snapshotId, long repositoryStateI final ActionListener afterCleanupsListener = new GroupedActionListener<>(ActionListener.wrap(() -> listener.onResponse(null)), 2); asyncCleanupUnlinkedRootAndIndicesBlobs(foundIndices, rootBlobs, updatedRepoData, afterCleanupsListener); - asyncCleanupUnlinkedShardLevelBlobs(snapshotId, writeShardMetadataAndComputeDeletesStep.result(), afterCleanupsListener); + asyncCleanupUnlinkedShardLevelBlobs(snapshotIds, writeShardMetaDataAndComputeDeletesStep.result(), afterCleanupsListener); }, listener::onFailure); } else { // Write the new repository data first (with the removed snapshot), using no shard generations - final RepositoryData updatedRepoData = repositoryData.removeSnapshot(snapshotId, ShardGenerations.EMPTY); + final RepositoryData updatedRepoData = repositoryData.removeSnapshots(snapshotIds, ShardGenerations.EMPTY); writeIndexGen(updatedRepoData, repositoryStateId, false, Function.identity(), ActionListener.wrap(v -> { // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion final ActionListener afterCleanupsListener = new GroupedActionListener<>(ActionListener.wrap(() -> listener.onResponse(null)), 2); asyncCleanupUnlinkedRootAndIndicesBlobs(foundIndices, rootBlobs, updatedRepoData, afterCleanupsListener); final StepListener> writeMetaAndComputeDeletesStep = new StepListener<>(); - writeUpdatedShardMetadataAndComputeDeletes(snapshotId, repositoryData, false, writeMetaAndComputeDeletesStep); + writeUpdatedShardMetaDataAndComputeDeletes(snapshotIds, repositoryData, false, writeMetaAndComputeDeletesStep); writeMetaAndComputeDeletesStep.whenComplete(deleteResults -> - asyncCleanupUnlinkedShardLevelBlobs(snapshotId, deleteResults, afterCleanupsListener), + asyncCleanupUnlinkedShardLevelBlobs(snapshotIds, deleteResults, afterCleanupsListener), afterCleanupsListener::onFailure); }, listener::onFailure)); } @@ -643,17 +632,18 @@ private void asyncCleanupUnlinkedRootAndIndicesBlobs(Map l -> cleanupStaleBlobs(foundIndices, rootBlobs, updatedRepoData, ActionListener.map(l, ignored -> null)))); } - private void asyncCleanupUnlinkedShardLevelBlobs(SnapshotId snapshotId, Collection deleteResults, + private void asyncCleanupUnlinkedShardLevelBlobs(Collection snapshotIds, + Collection deleteResults, ActionListener listener) { threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.wrap( listener, l -> { try { - blobContainer().deleteBlobsIgnoringIfNotExists(resolveFilesToDelete(snapshotId, deleteResults)); + blobContainer().deleteBlobsIgnoringIfNotExists(resolveFilesToDelete(snapshotIds, deleteResults)); l.onResponse(null); } catch (Exception e) { logger.warn( - () -> new ParameterizedMessage("[{}] Failed to delete some blobs during snapshot delete", snapshotId), + () -> new ParameterizedMessage("{} Failed to delete some blobs during snapshot delete", snapshotIds), e); throw e; } @@ -661,11 +651,11 @@ private void asyncCleanupUnlinkedShardLevelBlobs(SnapshotId snapshotId, Collecti } // updates the shard state metadata for shards of a snapshot that is to be deleted. Also computes the files to be cleaned up. - private void writeUpdatedShardMetadataAndComputeDeletes(SnapshotId snapshotId, RepositoryData oldRepositoryData, + private void writeUpdatedShardMetaDataAndComputeDeletes(Collection snapshotIds, RepositoryData oldRepositoryData, boolean useUUIDs, ActionListener> onAllShardsCompleted) { final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT); - final List indices = oldRepositoryData.indicesToUpdateAfterRemovingSnapshot(snapshotId); + final List indices = oldRepositoryData.indicesToUpdateAfterRemovingSnapshot(snapshotIds); if (indices.isEmpty()) { onAllShardsCompleted.onResponse(Collections.emptyList()); @@ -679,66 +669,72 @@ private void writeUpdatedShardMetadataAndComputeDeletes(SnapshotId snapshotId, R for (IndexId indexId : indices) { final Set survivingSnapshots = oldRepositoryData.getSnapshots(indexId).stream() - .filter(id -> id.equals(snapshotId) == false).collect(Collectors.toSet()); - executor.execute(ActionRunnable.wrap(deleteIndexMetadataListener, deleteIdxMetaListener -> { - final IndexMetadata indexMetadata; - try { - indexMetadata = getSnapshotIndexMetadata(snapshotId, indexId); - } catch (Exception ex) { - logger.warn(() -> - new ParameterizedMessage("[{}] [{}] failed to read metadata for index", snapshotId, indexId.getName()), ex); - // Just invoke the listener without any shard generations to count it down, this index will be cleaned up - // by the stale data cleanup in the end. - // TODO: Getting here means repository corruption. We should find a way of dealing with this instead of just ignoring - // it and letting the cleanup deal with it. - deleteIdxMetaListener.onResponse(null); + .filter(id -> snapshotIds.contains(id) == false).collect(Collectors.toSet()); + final StepListener> shardCountListener = new StepListener<>(); + final ActionListener allShardCountsListener = new GroupedActionListener<>(shardCountListener, snapshotIds.size()); + for (SnapshotId snapshotId : snapshotIds) { + executor.execute(ActionRunnable.supply(allShardCountsListener, () -> { + try { + return getSnapshotIndexMetadata(snapshotId, indexId).getNumberOfShards(); + } catch (Exception ex) { + logger.warn(() -> new ParameterizedMessage( + "[{}] [{}] failed to read metadata for index", snapshotId, indexId.getName()), ex); + // Just invoke the listener without any shard generations to count it down, this index will be cleaned up + // by the stale data cleanup in the end. + // TODO: Getting here means repository corruption. We should find a way of dealing with this instead of just + // ignoring it and letting the cleanup deal with it. + return null; + } + })); + } + shardCountListener.whenComplete(counts -> { + final int shardCount = counts.stream().mapToInt(i -> i).max().orElse(0); + if (shardCount == 0) { + deleteIndexMetadataListener.onResponse(null); return; } - final int shardCount = indexMetadata.getNumberOfShards(); - assert shardCount > 0 : "index did not have positive shard count, get [" + shardCount + "]"; // Listener for collecting the results of removing the snapshot from each shard's metadata in the current index final ActionListener allShardsListener = - new GroupedActionListener<>(deleteIdxMetaListener, shardCount); - final Index index = indexMetadata.getIndex(); - for (int shardId = 0; shardId < indexMetadata.getNumberOfShards(); shardId++) { - final ShardId shard = new ShardId(index, shardId); + new GroupedActionListener<>(deleteIndexMetadataListener, shardCount); + for (int shardId = 0; shardId < shardCount; shardId++) { + final int finalShardId = shardId; executor.execute(new AbstractRunnable() { @Override protected void doRun() throws Exception { - final BlobContainer shardContainer = shardContainer(indexId, shard); - final Set blobs = getShardBlobs(shard, shardContainer); + final BlobContainer shardContainer = shardContainer(indexId, finalShardId); + final Set blobs = shardContainer.listBlobs().keySet(); final BlobStoreIndexShardSnapshots blobStoreIndexShardSnapshots; final String newGen; if (useUUIDs) { newGen = UUIDs.randomBase64UUID(); blobStoreIndexShardSnapshots = buildBlobStoreIndexShardSnapshots(blobs, shardContainer, - oldRepositoryData.shardGenerations().getShardGen(indexId, shard.getId())).v1(); + oldRepositoryData.shardGenerations().getShardGen(indexId, finalShardId)).v1(); } else { - Tuple tuple = - buildBlobStoreIndexShardSnapshots(blobs, shardContainer); + Tuple tuple = buildBlobStoreIndexShardSnapshots(blobs, shardContainer); newGen = Long.toString(tuple.v2() + 1); blobStoreIndexShardSnapshots = tuple.v1(); } - allShardsListener.onResponse(deleteFromShardSnapshotMeta(survivingSnapshots, indexId, shard, snapshotId, - shardContainer, blobs, blobStoreIndexShardSnapshots, newGen)); + allShardsListener.onResponse(deleteFromShardSnapshotMeta(survivingSnapshots, indexId, finalShardId, + snapshotIds, shardContainer, blobs, blobStoreIndexShardSnapshots, newGen)); } @Override public void onFailure(Exception ex) { logger.warn( - () -> new ParameterizedMessage("[{}] failed to delete shard data for shard [{}][{}]", - snapshotId, indexId.getName(), shard.id()), ex); + () -> new ParameterizedMessage("{} failed to delete shard data for shard [{}][{}]", + snapshotIds, indexId.getName(), finalShardId), ex); // Just passing null here to count down the listener instead of failing it, the stale data left behind // here will be retried in the next delete or repository cleanup allShardsListener.onResponse(null); } }); } - })); + }, deleteIndexMetadataListener::onFailure); } } - private List resolveFilesToDelete(SnapshotId snapshotId, Collection deleteResults) { + private List resolveFilesToDelete(Collection snapshotIds, + Collection deleteResults) { final String basePath = basePath().buildAsString(); final int basePathLen = basePath.length(); return Stream.concat( @@ -747,8 +743,10 @@ private List resolveFilesToDelete(SnapshotId snapshotId, Collection shardPath + blob); }), - deleteResults.stream().map(shardResult -> shardResult.indexId).distinct().map(indexId -> - indexContainer(indexId).path().buildAsString() + globalMetadataFormat.blobName(snapshotId.getUUID())) + deleteResults.stream().map(shardResult -> shardResult.indexId).distinct().flatMap(indexId -> { + final String indexContainerPath = indexContainer(indexId).path().buildAsString(); + return snapshotIds.stream().map(snapshotId -> indexContainerPath + globalMetadataFormat.blobName(snapshotId.getUUID())); + }) ).map(absolutePath -> { assert absolutePath.startsWith(basePath); return absolutePath.substring(basePathLen); @@ -1940,9 +1938,10 @@ public String toString() { * Delete snapshot from shard level metadata. */ private ShardSnapshotMetaDeleteResult deleteFromShardSnapshotMeta(Set survivingSnapshots, IndexId indexId, - ShardId snapshotShardId, SnapshotId snapshotId, + int snapshotShardId, Collection snapshotIds, BlobContainer shardContainer, Set blobs, - BlobStoreIndexShardSnapshots snapshots, String indexGeneration) { + BlobStoreIndexShardSnapshots snapshots, + String indexGeneration) { // Build a list of snapshots that should be preserved List newSnapshotsList = new ArrayList<>(); final Set survivingSnapshotNames = survivingSnapshots.stream().map(SnapshotId::getName).collect(Collectors.toSet()); @@ -1953,18 +1952,17 @@ private ShardSnapshotMetaDeleteResult deleteFromShardSnapshotMeta(Set survivingSnapshotUUIDs = survivingSnapshots.stream().map(SnapshotId::getUUID).collect(Collectors.toSet()); - return new ShardSnapshotMetaDeleteResult(indexId, snapshotShardId.id(), indexGeneration, + return new ShardSnapshotMetaDeleteResult(indexId, snapshotShardId, indexGeneration, unusedBlobs(blobs, survivingSnapshotUUIDs, updatedSnapshots)); } } catch (IOException e) { - throw new IndexShardSnapshotFailedException(snapshotShardId, - "Failed to finalize snapshot deletion [" + snapshotId + "] with shard index [" - + indexShardSnapshotsFormat.blobName(indexGeneration) + "]", e); + throw new RepositoryException(metadata.name(), "Failed to finalize snapshot deletion " + snapshotIds + + " with shard index [" + indexShardSnapshotsFormat.blobName(indexGeneration) + "]", e); } } @@ -1975,16 +1973,6 @@ private void writeShardIndexBlob(BlobContainer shardContainer, String indexGener indexShardSnapshotsFormat.writeAtomic(updatedSnapshots, shardContainer, indexGeneration); } - private static Set getShardBlobs(final ShardId snapshotShardId, final BlobContainer shardContainer) { - final Set blobs; - try { - blobs = shardContainer.listBlobs().keySet(); - } catch (IOException e) { - throw new IndexShardSnapshotException(snapshotShardId, "Failed to list content of shard directory", e); - } - return blobs; - } - // Unused blobs are all previous index-, data- and meta-blobs and that are not referenced by the new index- as well as all // temporary blobs private static List unusedBlobs(Set blobs, Set survivingSnapshotUUIDs, diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/package-info.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/package-info.java index f35d5eebfafcc..d2a939792b75b 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/package-info.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/package-info.java @@ -215,7 +215,7 @@ *

Deleting a Snapshot

* *

Deleting a snapshot is an operation that is exclusively executed on the master node that runs through the following sequence of - * action when {@link org.elasticsearch.repositories.blobstore.BlobStoreRepository#deleteSnapshot} is invoked:

+ * action when {@link org.elasticsearch.repositories.blobstore.BlobStoreRepository#deleteSnapshots} is invoked:

* *
    *
  1. Get the current {@code RepositoryData} from the latest {@code index-N} blob at the repository root.
    2. diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteSnapshotAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteSnapshotAction.java index f74fc73c1f326..28f5770131110 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteSnapshotAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.Strings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; @@ -49,7 +50,8 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - DeleteSnapshotRequest deleteSnapshotRequest = deleteSnapshotRequest(request.param("repository"), request.param("snapshot")); + DeleteSnapshotRequest deleteSnapshotRequest = deleteSnapshotRequest(request.param("repository"), + Strings.splitStringByCommaToArray(request.param("snapshot"))); deleteSnapshotRequest.masterNodeTimeout(request.paramAsTime("master_timeout", deleteSnapshotRequest.masterNodeTimeout())); return channel -> client.admin().cluster().deleteSnapshot(deleteSnapshotRequest, new RestToXContentListener<>(channel)); } diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 7711d184c4038..831f2782c02ae 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -235,10 +235,10 @@ public ClusterState execute(ClusterState currentState) { // Check if the snapshot to restore is currently being deleted SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(SnapshotDeletionsInProgress.TYPE); if (deletionsInProgress != null - && deletionsInProgress.getEntries().stream().anyMatch(entry -> entry.getSnapshot().equals(snapshot))) { + && deletionsInProgress.getEntries().stream().anyMatch(entry -> entry.getSnapshots().contains(snapshotId))) { throw new ConcurrentSnapshotExecutionException(snapshot, "cannot restore a snapshot while a snapshot deletion is in-progress [" + - deletionsInProgress.getEntries().get(0).getSnapshot() + "]"); + deletionsInProgress.getEntries().get(0) + "]"); } // Updating cluster state diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index be663dd443473..d137f390d610f 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -61,6 +61,7 @@ import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractLifecycleComponent; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; @@ -84,10 +85,10 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -111,6 +112,8 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus public static final Version OLD_SNAPSHOT_FORMAT = Version.V_7_5_0; + public static final Version MULTI_DELETE_VERSION = Version.V_7_8_0; + private static final Logger logger = LogManager.getLogger(SnapshotsService.class); private final ClusterService clusterService; @@ -624,7 +627,8 @@ private void finalizeSnapshotDeletionFromPreviousMaster(ClusterState state) { if (deletionsInProgress != null && deletionsInProgress.hasDeletionsInProgress()) { assert deletionsInProgress.getEntries().size() == 1 : "only one in-progress deletion allowed per cluster"; SnapshotDeletionsInProgress.Entry entry = deletionsInProgress.getEntries().get(0); - deleteSnapshotFromRepository(entry.getSnapshot(), null, entry.repositoryStateId(), state.nodes().getMinNodeVersion()); + deleteSnapshotsFromRepository(entry.repository(), entry.getSnapshots(), null, entry.repositoryStateId(), + state.nodes().getMinNodeVersion()); } } @@ -992,17 +996,18 @@ private void failSnapshotCompletionListeners(Snapshot snapshot, Exception e) { } /** - * Deletes a snapshot from the repository or aborts a running snapshot. - * First checks if the snapshot is still running and if so cancels the snapshot and then deletes it from the repository. - * If the snapshot is not running, moves to trying to find a matching {@link Snapshot} for the given name in the repository and if - * one is found deletes it by invoking {@link #deleteCompletedSnapshot}. + * Deletes snapshots from the repository or aborts a running snapshot. + * If deleting a single snapshot, first checks if a snapshot is still running and if so cancels the snapshot and then deletes it from + * the repository. + * If the snapshot is not running or multiple snapshot names are given, moves to trying to find a matching {@link Snapshot}s for the + * given names in the repository and deletes them by invoking {@link #deleteCompletedSnapshots}. * * @param repositoryName repositoryName - * @param snapshotName snapshotName + * @param snapshotNames snapshotNames * @param listener listener */ - public void deleteSnapshot(final String repositoryName, final String snapshotName, final ActionListener listener) { - logger.info("deleting snapshot [{}] from repository [{}]", snapshotName, repositoryName); + public void deleteSnapshots(final String repositoryName, final Collection snapshotNames, final ActionListener listener) { + logger.info("deleting snapshots {} from repository [{}]", snapshotNames, repositoryName); clusterService.submitStateUpdateTask("delete snapshot", new ClusterStateUpdateTask(Priority.NORMAL) { @@ -1012,8 +1017,23 @@ public void deleteSnapshot(final String repositoryName, final String snapshotNam @Override public ClusterState execute(ClusterState currentState) { + if (snapshotNames.size() > 1 && currentState.nodes().getMinNodeVersion().before(MULTI_DELETE_VERSION)) { + throw new IllegalArgumentException("Deleting multiple snapshots in a single request is only supported in version [ " + + MULTI_DELETE_VERSION + "] but cluster contained node of version [" + currentState.nodes().getMinNodeVersion() + + "]"); + } final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE); - final SnapshotsInProgress.Entry snapshotEntry = findInProgressSnapshot(snapshots, snapshotName, repositoryName); + final SnapshotsInProgress.Entry snapshotEntry; + if (snapshotNames.size() == 1) { + final String snapshotName = snapshotNames.iterator().next(); + if (Regex.isSimpleMatchPattern(snapshotName)) { + snapshotEntry = null; + } else { + snapshotEntry = findInProgressSnapshot(snapshots, snapshotName, repositoryName); + } + } else { + snapshotEntry = null; + } if (snapshotEntry == null) { return currentState; } @@ -1082,29 +1102,17 @@ public void onFailure(String source, Exception e) { public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { if (runningSnapshot == null) { threadPool.generic().execute(ActionRunnable.wrap(listener, l -> - repositoriesService.repository(repositoryName).getRepositoryData( - ActionListener.wrap(repositoryData -> { - Optional matchedEntry = repositoryData.getSnapshotIds() - .stream() - .filter(s -> s.getName().equals(snapshotName)) - .findFirst(); - // If we can't find the snapshot by the given name in the repository at all or if the snapshot we find in - // the repository is not the one we expected to find when waiting for a finishing snapshot we fail. - if (matchedEntry.isPresent()) { - deleteCompletedSnapshot( - new Snapshot(repositoryName, matchedEntry.get()), repositoryData.getGenId(), Priority.NORMAL, l); - } else { - l.onFailure(new SnapshotMissingException(repositoryName, snapshotName)); - } - }, l::onFailure)))); + repositoriesService.repository(repositoryName).getRepositoryData(ActionListener.wrap(repositoryData -> + deleteCompletedSnapshots(matchingSnapshotIds(repositoryData, snapshotNames, repositoryName), + repositoryName, repositoryData.getGenId(), Priority.NORMAL, l), l::onFailure)))); return; } logger.trace("adding snapshot completion listener to wait for deleted snapshot to finish"); addListener(runningSnapshot, ActionListener.wrap( result -> { logger.debug("deleted snapshot completed - deleting files"); - deleteCompletedSnapshot( - new Snapshot(repositoryName, result.v2().snapshotId()), result.v1().getGenId(), Priority.IMMEDIATE, listener); + deleteCompletedSnapshots(Collections.singletonList(result.v2().snapshotId()), repositoryName, + result.v1().getGenId(), Priority.IMMEDIATE, listener); }, e -> { if (abortedDuringInit) { @@ -1128,6 +1136,30 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS }); } + private static List matchingSnapshotIds(RepositoryData repositoryData, Collection snapshotsOrPatterns, + String repositoryName) { + final Map allSnapshotIds = repositoryData.getSnapshotIds().stream().collect( + Collectors.toMap(SnapshotId::getName, Function.identity())); + final Set foundSnapshots = new HashSet<>(); + for (String snapshotOrPattern : snapshotsOrPatterns) { + if (Regex.isSimpleMatchPattern(snapshotOrPattern) == false) { + final SnapshotId foundId = allSnapshotIds.get(snapshotOrPattern); + if (foundId == null) { + throw new SnapshotMissingException(repositoryName, snapshotOrPattern); + } else { + foundSnapshots.add(allSnapshotIds.get(snapshotOrPattern)); + } + } else { + for (Map.Entry entry : allSnapshotIds.entrySet()) { + if (Regex.simpleMatch(snapshotOrPattern, entry.getKey())) { + foundSnapshots.add(entry.getValue()); + } + } + } + } + return Collections.unmodifiableList(new ArrayList<>(foundSnapshots)); + } + // Return in-progress snapshot entry by name and repository in the given cluster state or null if none is found @Nullable private static SnapshotsInProgress.Entry findInProgressSnapshot(@Nullable SnapshotsInProgress snapshots, String snapshotName, @@ -1147,27 +1179,33 @@ private static SnapshotsInProgress.Entry findInProgressSnapshot(@Nullable Snapsh } /** - * Deletes a snapshot that is assumed to be in the repository and not tracked as in-progress in the cluster state. + * Deletes snapshots that are assumed to be in the repository and not tracked as in-progress in the cluster state. * - * @param snapshot Snapshot to delete + * @param snapshotIds Snapshots to delete + * @param repoName Repository name * @param repositoryStateId Repository generation to base the delete on * @param listener Listener to complete when done */ - private void deleteCompletedSnapshot(Snapshot snapshot, long repositoryStateId, Priority priority, ActionListener listener) { - logger.debug("deleting snapshot [{}] assuming repository generation [{}] and with priority [{}]", snapshot, repositoryStateId, + private void deleteCompletedSnapshots(List snapshotIds, String repoName, long repositoryStateId, Priority priority, + ActionListener listener) { + if (snapshotIds.isEmpty()) { + listener.onResponse(null); + return; + } + logger.debug("deleting snapshots {} assuming repository generation [{}] and with priority [{}]", snapshotIds, repositoryStateId, priority); clusterService.submitStateUpdateTask("delete snapshot", new ClusterStateUpdateTask(priority) { @Override public ClusterState execute(ClusterState currentState) { SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(SnapshotDeletionsInProgress.TYPE); if (deletionsInProgress != null && deletionsInProgress.hasDeletionsInProgress()) { - throw new ConcurrentSnapshotExecutionException(snapshot, + throw new ConcurrentSnapshotExecutionException(new Snapshot(repoName, snapshotIds.get(0)), "cannot delete - another snapshot is currently being deleted in [" + deletionsInProgress + "]"); } final RepositoryCleanupInProgress repositoryCleanupInProgress = currentState.custom(RepositoryCleanupInProgress.TYPE); if (repositoryCleanupInProgress != null && repositoryCleanupInProgress.hasCleanupInProgress()) { - throw new ConcurrentSnapshotExecutionException(snapshot.getRepository(), snapshot.getSnapshotId().getName(), - "cannot delete snapshot while a repository cleanup is in-progress in [" + repositoryCleanupInProgress + "]"); + throw new ConcurrentSnapshotExecutionException(new Snapshot(repoName, snapshotIds.get(0)), + "cannot delete snapshots while a repository cleanup is in-progress in [" + repositoryCleanupInProgress + "]"); } RestoreInProgress restoreInProgress = currentState.custom(RestoreInProgress.TYPE); if (restoreInProgress != null) { @@ -1176,8 +1214,8 @@ public ClusterState execute(ClusterState currentState) { // and the files the restore depends on would all be gone for (RestoreInProgress.Entry entry : restoreInProgress) { - if (entry.snapshot().equals(snapshot)) { - throw new ConcurrentSnapshotExecutionException(snapshot, + if (repoName.equals(entry.snapshot().getRepository()) && snapshotIds.contains(entry.snapshot().getSnapshotId())) { + throw new ConcurrentSnapshotExecutionException(new Snapshot(repoName, snapshotIds.get(0)), "cannot delete snapshot during a restore in progress in [" + restoreInProgress + "]"); } } @@ -1185,13 +1223,15 @@ public ClusterState execute(ClusterState currentState) { SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE); if (snapshots != null && snapshots.entries().isEmpty() == false) { // However other snapshots are running - cannot continue - throw new ConcurrentSnapshotExecutionException(snapshot, "another snapshot is currently running cannot delete"); + throw new ConcurrentSnapshotExecutionException( + repoName, snapshotIds.toString(), "another snapshot is currently running cannot delete"); } // add the snapshot deletion to the cluster state SnapshotDeletionsInProgress.Entry entry = new SnapshotDeletionsInProgress.Entry( - snapshot, - threadPool.absoluteTimeInMillis(), - repositoryStateId + snapshotIds, + repoName, + threadPool.absoluteTimeInMillis(), + repositoryStateId ); if (deletionsInProgress != null) { deletionsInProgress = deletionsInProgress.withAddedEntry(entry); @@ -1208,7 +1248,7 @@ public void onFailure(String source, Exception e) { @Override public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { - deleteSnapshotFromRepository(snapshot, listener, repositoryStateId, newState.nodes().getMinNodeVersion()); + deleteSnapshotsFromRepository(repoName, snapshotIds, listener, repositoryStateId, newState.nodes().getMinNodeVersion()); } }); } @@ -1226,12 +1266,12 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS * @return minimum node version that must still be able to read the repository metadata */ public Version minCompatibleVersion(Version minNodeVersion, String repositoryName, RepositoryData repositoryData, - @Nullable SnapshotId excluded) { + @Nullable Collection excluded) { Version minCompatVersion = minNodeVersion; final Collection snapshotIds = repositoryData.getSnapshotIds(); final Repository repository = repositoriesService.repository(repositoryName); - for (SnapshotId snapshotId : - snapshotIds.stream().filter(snapshotId -> snapshotId.equals(excluded) == false).collect(Collectors.toList())) { + for (SnapshotId snapshotId : snapshotIds.stream().filter(excluded == null ? sn -> true : sn -> excluded.contains(sn) == false) + .collect(Collectors.toList())) { final Version known = repositoryData.getVersion(snapshotId); // If we don't have the version cached in the repository data yet we load it from the snapshot info blobs if (known == null) { @@ -1270,30 +1310,31 @@ public static boolean useShardGenerations(Version repositoryMetaVersion) { /** * Deletes snapshot from repository * - * @param snapshot snapshot - * @param listener listener + * @param repoName repository name + * @param snapshotIds snapshot ids + * @param listener listener * @param repositoryStateId the unique id representing the state of the repository at the time the deletion began - * @param minNodeVersion minimum node version in the cluster + * @param minNodeVersion minimum node version in the cluster */ - private void deleteSnapshotFromRepository(Snapshot snapshot, @Nullable ActionListener listener, long repositoryStateId, - Version minNodeVersion) { + private void deleteSnapshotsFromRepository(String repoName, Collection snapshotIds, @Nullable ActionListener listener, + long repositoryStateId, Version minNodeVersion) { threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.wrap(listener, l -> { - Repository repository = repositoriesService.repository(snapshot.getRepository()); - repository.getRepositoryData(ActionListener.wrap(repositoryData -> repository.deleteSnapshot(snapshot.getSnapshotId(), + Repository repository = repositoriesService.repository(repoName); + repository.getRepositoryData(ActionListener.wrap(repositoryData -> repository.deleteSnapshots(snapshotIds, repositoryStateId, - minCompatibleVersion(minNodeVersion, snapshot.getRepository(), repositoryData, snapshot.getSnapshotId()), + minCompatibleVersion(minNodeVersion, repoName, repositoryData, snapshotIds), ActionListener.wrap(v -> { - logger.info("snapshot [{}] deleted", snapshot); - removeSnapshotDeletionFromClusterState(snapshot, null, l); - }, ex -> removeSnapshotDeletionFromClusterState(snapshot, ex, l) - )), ex -> removeSnapshotDeletionFromClusterState(snapshot, ex, l))); + logger.info("snapshots {} deleted", snapshotIds); + removeSnapshotDeletionFromClusterState(snapshotIds, null, l); + }, ex -> removeSnapshotDeletionFromClusterState(snapshotIds, ex, l) + )), ex -> removeSnapshotDeletionFromClusterState(snapshotIds, ex, l))); })); } /** * Removes the snapshot deletion from {@link SnapshotDeletionsInProgress} in the cluster state. */ - private void removeSnapshotDeletionFromClusterState(final Snapshot snapshot, @Nullable final Exception failure, + private void removeSnapshotDeletionFromClusterState(final Collection snapshotIds, @Nullable final Exception failure, @Nullable final ActionListener listener) { clusterService.submitStateUpdateTask("remove snapshot deletion metadata", new ClusterStateUpdateTask() { @Override @@ -1316,7 +1357,7 @@ public ClusterState execute(ClusterState currentState) { @Override public void onFailure(String source, Exception e) { - logger.warn(() -> new ParameterizedMessage("[{}] failed to remove snapshot deletion metadata", snapshot), e); + logger.warn(() -> new ParameterizedMessage("{} failed to remove snapshot deletion metadata", snapshotIds), e); if (listener != null) { listener.onFailure(e); } @@ -1328,7 +1369,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS if (failure != null) { listener.onFailure(failure); } else { - logger.info("Successfully deleted snapshot [{}]", snapshot); + logger.info("Successfully deleted snapshots {}", snapshotIds); listener.onResponse(null); } } diff --git a/server/src/main/java/org/elasticsearch/snapshots/package-info.java b/server/src/main/java/org/elasticsearch/snapshots/package-info.java index 010e63eae6b4f..d4a31ad41352c 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/package-info.java +++ b/server/src/main/java/org/elasticsearch/snapshots/package-info.java @@ -97,7 +97,7 @@ * {@link org.elasticsearch.cluster.SnapshotDeletionsInProgress}. * *
  2. Once the cluster state contains the deletion entry in {@code SnapshotDeletionsInProgress} the {@code SnapshotsService} will invoke - * {@link org.elasticsearch.repositories.Repository#deleteSnapshot} for the given snapshot, which will remove files associated with the + * {@link org.elasticsearch.repositories.Repository#deleteSnapshots} for the given snapshot, which will remove files associated with the * snapshot from the repository as well as update its meta-data to reflect the deletion of the snapshot.
    3. * *
  3. After the deletion of the snapshot's data from the repository finishes, the {@code SnapshotsService} will submit a cluster state diff --git a/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java b/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java index b19e0e0cdb7cb..06b1b5ba9f965 100644 --- a/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java @@ -123,7 +123,7 @@ public void testSnapshotDeletionsInProgressSerialization() throws Exception { .putCustom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.newInstance( new SnapshotDeletionsInProgress.Entry( - new Snapshot("repo1", new SnapshotId("snap1", UUIDs.randomBase64UUID())), + Collections.singletonList(new SnapshotId("snap1", UUIDs.randomBase64UUID())), "repo1", randomNonNegativeLong(), randomNonNegativeLong()) )); if (includeRestore) { diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java index 60481e4272944..915d5e8a58820 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java @@ -49,6 +49,7 @@ import org.elasticsearch.transport.TransportService; import java.io.IOException; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -172,8 +173,8 @@ public void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations indices, lo } @Override - public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, Version repositoryMetaVersion, - ActionListener listener) { + public void deleteSnapshots(Collection snapshotIds, long repositoryStateId, Version repositoryMetaVersion, + ActionListener listener) { listener.onResponse(null); } diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java index 5c7422d1fb592..4bd212b09604b 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java @@ -67,7 +67,8 @@ public void testIndicesToUpdateAfterRemovingSnapshot() { final List snapshotIds = repositoryData.getSnapshots(index); return snapshotIds.contains(randomSnapshot) && snapshotIds.size() > 1; }).toArray(IndexId[]::new); - assertThat(repositoryData.indicesToUpdateAfterRemovingSnapshot(randomSnapshot), containsInAnyOrder(indicesToUpdate)); + assertThat(repositoryData.indicesToUpdateAfterRemovingSnapshot( + Collections.singleton(randomSnapshot)), containsInAnyOrder(indicesToUpdate)); } public void testXContent() throws IOException { @@ -152,7 +153,7 @@ public void testRemoveSnapshot() { List snapshotIds = new ArrayList<>(repositoryData.getSnapshotIds()); assertThat(snapshotIds.size(), greaterThan(0)); SnapshotId removedSnapshotId = snapshotIds.remove(randomIntBetween(0, snapshotIds.size() - 1)); - RepositoryData newRepositoryData = repositoryData.removeSnapshot(removedSnapshotId, ShardGenerations.EMPTY); + RepositoryData newRepositoryData = repositoryData.removeSnapshots(Collections.singleton(removedSnapshotId), ShardGenerations.EMPTY); // make sure the repository data's indices no longer contain the removed snapshot for (final IndexId indexId : newRepositoryData.getIndices().values()) { assertFalse(newRepositoryData.getSnapshots(indexId).contains(removedSnapshotId)); diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java index 8347ba5a977b9..fec8b6c15359f 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -184,8 +184,8 @@ public void testIndexGenerationalFiles() throws Exception { assertThat(repository.readSnapshotIndexLatestBlob(), equalTo(expectedGeneration + 1L)); // removing a snapshot and writing to a new index generational file - repositoryData = ESBlobStoreRepositoryIntegTestCase.getRepositoryData(repository).removeSnapshot( - repositoryData.getSnapshotIds().iterator().next(), ShardGenerations.EMPTY); + repositoryData = ESBlobStoreRepositoryIntegTestCase.getRepositoryData(repository).removeSnapshots( + Collections.singleton(repositoryData.getSnapshotIds().iterator().next()), ShardGenerations.EMPTY); writeIndexGen(repository, repositoryData, repositoryData.getGenId()); assertEquals(ESBlobStoreRepositoryIntegTestCase.getRepositoryData(repository), repositoryData); assertThat(repository.latestIndexBlobId(), equalTo(expectedGeneration + 2L)); diff --git a/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index e7cee3e883841..f54bf1d2d9b19 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -134,6 +134,7 @@ import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasSize; @@ -1277,8 +1278,14 @@ public void testDeleteSnapshot() throws Exception { int numberOfFilesBeforeDeletion = numberOfFiles(repo); logger.info("--> delete all snapshots except the first one and last one"); - for (int i = 1; i < numberOfSnapshots - 1; i++) { - client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap-" + i).get(); + + if (randomBoolean()) { + for (int i = 1; i < numberOfSnapshots - 1; i++) { + client.admin().cluster().prepareDeleteSnapshot("test-repo", new String[]{"test-snap-" + i}).get(); + } + } else { + client.admin().cluster().prepareDeleteSnapshot( + "test-repo", IntStream.range(1, numberOfSnapshots - 1).mapToObj(i -> "test-snap-" + i).toArray(String[]::new)).get(); } int numberOfFilesAfterDeletion = numberOfFiles(repo); @@ -3691,6 +3698,40 @@ public void testSnapshotDifferentIndicesBySameName() { assertHitCount(client().prepareSearch("restored-3").setSize(0).get(), expectedCount); } + public void testBulkDeleteWithOverlappingPatterns() { + final int numberOfSnapshots = between(5, 15); + Path repo = randomRepoPath(); + logger.info("--> creating repository at {}", repo.toAbsolutePath()); + assertAcked(client().admin().cluster().preparePutRepository("test-repo") + .setType("fs").setSettings(Settings.builder() + .put("location", repo) + .put("compress", false) + .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES))); + + final String[] indices = {"test-idx-1", "test-idx-2", "test-idx-3"}; + createIndex(indices); + ensureGreen(); + + logger.info("--> creating {} snapshots ", numberOfSnapshots); + for (int i = 0; i < numberOfSnapshots; i++) { + for (int j = 0; j < 10; j++) { + index(randomFrom(indices), Integer.toString(i * 10 + j), "foo", "bar" + i * 10 + j); + } + refresh(); + logger.info("--> snapshot {}", i); + CreateSnapshotResponse createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-" + i) + .setWaitForCompletion(true).get(); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), + equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); + } + + logger.info("--> deleting all snapshots"); + client().admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap-*", "*").get(); + final GetSnapshotsResponse getSnapshotsResponse = client().admin().cluster().prepareGetSnapshots("test-repo").get(); + assertThat(getSnapshotsResponse.getSnapshots(), empty()); + } + private void verifySnapshotInfo(final GetSnapshotsResponse response, final Map> indicesPerSnapshot) { for (SnapshotInfo snapshotInfo : response.getSnapshots()) { final List expected = snapshotInfo.indices(); diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java index 99b72aa5a425a..d479676e40ad9 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java @@ -40,6 +40,7 @@ import org.elasticsearch.snapshots.SnapshotShardFailure; import java.io.IOException; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -109,8 +110,8 @@ public void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenera } @Override - public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, Version repositoryMetaVersion, - ActionListener listener) { + public void deleteSnapshots(Collection snapshotIds, long repositoryStateId, Version repositoryMetaVersion, + ActionListener listener) { listener.onResponse(null); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index 9610852385d3a..b056f9c006ff6 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -85,6 +85,7 @@ import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; @@ -269,8 +270,8 @@ public void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenera } @Override - public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, Version repositoryMetaVersion, - ActionListener listener) { + public void deleteSnapshots(Collection snapshotIds, long repositoryStateId, Version repositoryMetaVersion, + ActionListener listener) { throw new UnsupportedOperationException("Unsupported for repository of type: " + TYPE); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CleanupSnapshotStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CleanupSnapshotStepTests.java index fd3e736880618..1ac58e4eb92d7 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CleanupSnapshotStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CleanupSnapshotStepTests.java @@ -21,7 +21,7 @@ import java.util.Map; import static org.elasticsearch.xpack.core.ilm.AbstractStepMasterTimeoutTestCase.emptyClusterState; -import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.is; public class CleanupSnapshotStepTests extends AbstractStepTestCase { @@ -149,7 +149,7 @@ protected void ActionListener listener) { assertThat(action.name(), is(DeleteSnapshotAction.NAME)); assertTrue(request instanceof DeleteSnapshotRequest); - assertThat(((DeleteSnapshotRequest) request).snapshot(), equalTo(expectedSnapshotName)); + assertThat(((DeleteSnapshotRequest) request).snapshots(), arrayContaining(expectedSnapshotName)); } }; } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotRetentionTask.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotRetentionTask.java index e150090b53483..8c54943c5a2f7 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotRetentionTask.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotRetentionTask.java @@ -379,6 +379,8 @@ void deleteSnapshots(Map> snapshotsToDelete, for (SnapshotInfo info : snapshots) { final String policyId = getPolicyId(info); final long deleteStartTime = nowNanoSupplier.getAsLong(); + // TODO: Use snapshot multi-delete instead of this loop if all nodes in the cluster support it + // i.e are newer or equal to SnapshotsService#MULTI_DELETE_VERSION deleteSnapshot(policyId, repo, info.snapshotId(), slmStats, ActionListener.wrap(acknowledgedResponse -> { deleted.incrementAndGet(); if (acknowledgedResponse.isAcknowledged()) { diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java index 2cbc2e02cddaf..b799a3c00b13b 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java @@ -354,7 +354,8 @@ public void testOkToDeleteSnapshots() { assertThat(SnapshotRetentionTask.okayToDeleteSnapshots(state), equalTo(false)); SnapshotDeletionsInProgress delInProgress = new SnapshotDeletionsInProgress( - Collections.singletonList(new SnapshotDeletionsInProgress.Entry(snapshot, 0, 0))); + Collections.singletonList(new SnapshotDeletionsInProgress.Entry( + Collections.singletonList(snapshot.getSnapshotId()), snapshot.getRepository(), 0, 0))); state = ClusterState.builder(new ClusterName("cluster")) .putCustom(SnapshotDeletionsInProgress.TYPE, delInProgress) .build();