diff --git a/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc index e2c37c1c1731c..621161d80d6bb 100644 --- a/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc @@ -140,9 +140,15 @@ Allows setting a sort order for the result. Defaults to `start_time`, i.e. sorti (Optional, string) Sort order. Valid values are `asc` for ascending and `desc` for descending order. Defaults to `asc`, meaning ascending order. +`from_sort_value`:: +(Optional, string) +Value of the current sort column at which to start retrieval. Can either be a string snapshot- or repository name when sorting by +snapshot or repository name, a millisecond time value or a number when sorting by index- or shard count. + `after`:: (Optional, string) -Offset identifier to start pagination from as returned by the `next` field in the response body. +Offset identifier to start pagination from as returned by the `next` field in the response body. Using this parameter is mutually exclusive +with using the `from_sort_value` parameter. `offset`:: (Optional, integer) @@ -158,11 +164,11 @@ created by an SLM policy but not those snapshots that were not created by an SLM policy you can use the special pattern `_none` that will match all snapshots without an SLM policy. NOTE: The `after` parameter and `next` field allow for iterating through snapshots with some consistency guarantees regarding concurrent -creation or deletion of snapshots. It is guaranteed that any snapshot that exists at the beginning of the iteration and not concurrently +creation or deletion of snapshots. It is guaranteed that any snapshot that exists at the beginning of the iteration and is not concurrently deleted will be seen during the iteration. Snapshots concurrently created may be seen during an iteration. -NOTE: The parameters `size`, `order`, `after`, `offset`, `slm_policy_filter` and `sort` are not supported when using `verbose=false` and -the sort order for requests with `verbose=false` is undefined. +NOTE: The parameters `size`, `order`, `after`, `from_sort_value`, `offset`, `slm_policy_filter` and `sort` are not supported when using +`verbose=false` and the sort order for requests with `verbose=false` is undefined. [role="child_attributes"] [[get-snapshot-api-response-body]] @@ -628,3 +634,194 @@ The API returns the following response: // TESTRESPONSE[s/"end_time_in_millis": 1593094752019/"end_time_in_millis": $body.snapshots.1.end_time_in_millis/] // TESTRESPONSE[s/"duration_in_millis": 0/"duration_in_millis": $body.snapshots.0.duration_in_millis/] // TESTRESPONSE[s/"duration_in_millis": 1/"duration_in_millis": $body.snapshots.1.duration_in_millis/] + + +The following request returns information for all snapshots that come after `snapshot_2` when sorted by snapshot name in the default +ascending order. + +[source,console] +---- +GET /_snapshot/my_repository/*?sort=name&from_sort_value=snapshot_2 +---- + +The API returns the following response: + +[source,console-result] +---- +{ + "snapshots": [ + { + "snapshot": "snapshot_2", + "uuid": "vdRctLCxSketdKb54xw67g", + "repository": "my_repository", + "version_id": , + "version": , + "indices": [], + "data_streams": [], + "feature_states": [], + "include_global_state": true, + "state": "SUCCESS", + "start_time": "2020-07-06T21:55:18.130Z", + "start_time_in_millis": 1593093628851, + "end_time": "2020-07-06T21:55:18.130Z", + "end_time_in_millis": 1593094752019, + "duration_in_millis": 1, + "failures": [], + "shards": { + "total": 0, + "failed": 0, + "successful": 0 + } + }, + { + "snapshot": "snapshot_3", + "uuid": "dRctdKb54xw67gvLCxSket", + "repository": "my_repository", + "version_id": , + "version": , + "indices": [], + "data_streams": [], + "feature_states": [], + "include_global_state": true, + "state": "SUCCESS", + "start_time": "2020-07-06T21:55:18.129Z", + "start_time_in_millis": 1593093628850, + "end_time": "2020-07-06T21:55:18.129Z", + "end_time_in_millis": 1593094752018, + "duration_in_millis": 0, + "failures": [], + "shards": { + "total": 0, + "failed": 0, + "successful": 0 + } + } + ], + "total": 2, + "remaining": 0 +} +---- +// TESTRESPONSE[s/"uuid": "vdRctLCxSketdKb54xw67g"/"uuid": $body.snapshots.0.uuid/] +// TESTRESPONSE[s/"uuid": "dRctdKb54xw67gvLCxSket"/"uuid": $body.snapshots.1.uuid/] +// TESTRESPONSE[s/"version_id": /"version_id": $body.snapshots.0.version_id/] +// TESTRESPONSE[s/"version": /"version": $body.snapshots.0.version/] +// TESTRESPONSE[s/"start_time": "2020-07-06T21:55:18.130Z"/"start_time": $body.snapshots.0.start_time/] +// TESTRESPONSE[s/"start_time": "2020-07-06T21:55:18.129Z"/"start_time": $body.snapshots.1.start_time/] +// TESTRESPONSE[s/"start_time_in_millis": 1593093628851/"start_time_in_millis": $body.snapshots.0.start_time_in_millis/] +// TESTRESPONSE[s/"start_time_in_millis": 1593093628850/"start_time_in_millis": $body.snapshots.1.start_time_in_millis/] +// TESTRESPONSE[s/"end_time": "2020-07-06T21:55:18.130Z"/"end_time": $body.snapshots.0.end_time/] +// TESTRESPONSE[s/"end_time": "2020-07-06T21:55:18.129Z"/"end_time": $body.snapshots.1.end_time/] +// TESTRESPONSE[s/"end_time_in_millis": 1593094752019/"end_time_in_millis": $body.snapshots.0.end_time_in_millis/] +// TESTRESPONSE[s/"end_time_in_millis": 1593094752018/"end_time_in_millis": $body.snapshots.1.end_time_in_millis/] +// TESTRESPONSE[s/"duration_in_millis": 1/"duration_in_millis": $body.snapshots.0.duration_in_millis/] +// TESTRESPONSE[s/"duration_in_millis": 0/"duration_in_millis": $body.snapshots.1.duration_in_millis/] + + +The following request returns information for all snapshots with names starting with `snapshot_` and that started on or after timestamp +`1577833200000` (Jan 1st 2020) when sorted by snapshot start time in the default ascending order. + +[source,console] +---- +GET /_snapshot/my_repository/snapshot_*?sort=start_time&from_sort_value=1577833200000 +---- + +The API returns the following response: + +[source,console-result] +---- +{ + "snapshots": [ + { + "snapshot": "snapshot_1", + "uuid": "dKb54xw67gvdRctLCxSket", + "repository": "my_repository", + "version_id": , + "version": , + "indices": [], + "data_streams": [], + "feature_states": [], + "include_global_state": true, + "state": "SUCCESS", + "start_time": "2020-07-06T21:55:18.128Z", + "start_time_in_millis": 1593093628849, + "end_time": "2020-07-06T21:55:18.129Z", + "end_time_in_millis": 1593093628850, + "duration_in_millis": 1, + "failures": [], + "shards": { + "total": 0, + "failed": 0, + "successful": 0 + } + }, + { + "snapshot": "snapshot_2", + "uuid": "vdRctLCxSketdKb54xw67g", + "repository": "my_repository", + "version_id": , + "version": , + "indices": [], + "data_streams": [], + "feature_states": [], + "include_global_state": true, + "state": "SUCCESS", + "start_time": "2020-07-06T21:55:18.130Z", + "start_time_in_millis": 1593093628851, + "end_time": "2020-07-06T21:55:18.130Z", + "end_time_in_millis": 1593093628851, + "duration_in_millis": 0, + "failures": [], + "shards": { + "total": 0, + "failed": 0, + "successful": 0 + } + }, + { + "snapshot": "snapshot_3", + "uuid": "dRctdKb54xw67gvLCxSket", + "repository": "my_repository", + "version_id": , + "version": , + "indices": [], + "data_streams": [], + "feature_states": [], + "include_global_state": true, + "state": "SUCCESS", + "start_time": "2020-07-06T21:55:18.131Z", + "start_time_in_millis": 1593093628852, + "end_time": "2020-07-06T21:55:18.135Z", + "end_time_in_millis": 1593093628856, + "duration_in_millis": 4, + "failures": [], + "shards": { + "total": 0, + "failed": 0, + "successful": 0 + } + } + ], + "total": 3, + "remaining": 0 +} +---- +// TESTRESPONSE[s/"uuid": "dKb54xw67gvdRctLCxSket"/"uuid": $body.snapshots.0.uuid/] +// TESTRESPONSE[s/"uuid": "vdRctLCxSketdKb54xw67g"/"uuid": $body.snapshots.1.uuid/] +// TESTRESPONSE[s/"uuid": "dRctdKb54xw67gvLCxSket"/"uuid": $body.snapshots.2.uuid/] +// TESTRESPONSE[s/"version_id": /"version_id": $body.snapshots.0.version_id/] +// TESTRESPONSE[s/"version": /"version": $body.snapshots.0.version/] +// TESTRESPONSE[s/"start_time": "2020-07-06T21:55:18.128Z"/"start_time": $body.snapshots.0.start_time/] +// TESTRESPONSE[s/"start_time": "2020-07-06T21:55:18.130Z"/"start_time": $body.snapshots.1.start_time/] +// TESTRESPONSE[s/"start_time": "2020-07-06T21:55:18.131Z"/"start_time": $body.snapshots.2.start_time/] +// TESTRESPONSE[s/"start_time_in_millis": 1593093628849/"start_time_in_millis": $body.snapshots.0.start_time_in_millis/] +// TESTRESPONSE[s/"start_time_in_millis": 1593093628851/"start_time_in_millis": $body.snapshots.1.start_time_in_millis/] +// TESTRESPONSE[s/"start_time_in_millis": 1593093628852/"start_time_in_millis": $body.snapshots.2.start_time_in_millis/] +// TESTRESPONSE[s/"end_time": "2020-07-06T21:55:18.129Z"/"end_time": $body.snapshots.0.end_time/] +// TESTRESPONSE[s/"end_time": "2020-07-06T21:55:18.130Z"/"end_time": $body.snapshots.1.end_time/] +// TESTRESPONSE[s/"end_time": "2020-07-06T21:55:18.135Z"/"end_time": $body.snapshots.2.end_time/] +// TESTRESPONSE[s/"end_time_in_millis": 1593093628850/"end_time_in_millis": $body.snapshots.0.end_time_in_millis/] +// TESTRESPONSE[s/"end_time_in_millis": 1593093628851/"end_time_in_millis": $body.snapshots.1.end_time_in_millis/] +// TESTRESPONSE[s/"end_time_in_millis": 1593093628856/"end_time_in_millis": $body.snapshots.2.end_time_in_millis/] +// TESTRESPONSE[s/"duration_in_millis": 1/"duration_in_millis": $body.snapshots.0.duration_in_millis/] +// TESTRESPONSE[s/"duration_in_millis": 0/"duration_in_millis": $body.snapshots.1.duration_in_millis/] +// TESTRESPONSE[s/"duration_in_millis": 4/"duration_in_millis": $body.snapshots.2.duration_in_millis/] diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/snapshots/RestGetSnapshotsIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/snapshots/RestGetSnapshotsIT.java index 637b8254101c2..d2c2a41bb7c15 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/snapshots/RestGetSnapshotsIT.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/snapshots/RestGetSnapshotsIT.java @@ -33,8 +33,11 @@ import java.util.Collection; import java.util.HashSet; import java.util.List; +import java.util.Set; import static org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase.assertSnapshotListSorted; +import static org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase.matchAllPattern; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.in; import static org.hamcrest.Matchers.is; @@ -242,6 +245,77 @@ public void testFilterBySLMPolicy() throws Exception { ); } + public void testSortAfterStartTime() throws Exception { + final String repoName = "test-repo"; + AbstractSnapshotIntegTestCase.createRepository(logger, repoName, "fs"); + final HashSet startTimes = new HashSet<>(); + final SnapshotInfo snapshot1 = createFullSnapshotWithUniqueStartTime(repoName, "snapshot-1", startTimes); + final SnapshotInfo snapshot2 = createFullSnapshotWithUniqueStartTime(repoName, "snapshot-2", startTimes); + final SnapshotInfo snapshot3 = createFullSnapshotWithUniqueStartTime(repoName, "snapshot-3", startTimes); + + final List allSnapshotInfo = clusterAdmin().prepareGetSnapshots(matchAllPattern()) + .setSnapshots(matchAllPattern()) + .setSort(GetSnapshotsRequest.SortBy.START_TIME) + .get() + .getSnapshots(); + assertThat(allSnapshotInfo, is(org.elasticsearch.core.List.of(snapshot1, snapshot2, snapshot3))); + + final long startTime1 = snapshot1.startTime(); + final long startTime2 = snapshot2.startTime(); + final long startTime3 = snapshot3.startTime(); + + assertThat(allAfterStartTimeAscending(startTime1 - 1), is(allSnapshotInfo)); + assertThat(allAfterStartTimeAscending(startTime1), is(allSnapshotInfo)); + assertThat(allAfterStartTimeAscending(startTime2), is(org.elasticsearch.core.List.of(snapshot2, snapshot3))); + assertThat(allAfterStartTimeAscending(startTime3), is(org.elasticsearch.core.List.of(snapshot3))); + assertThat(allAfterStartTimeAscending(startTime3 + 1), empty()); + + final List allSnapshotInfoDesc = clusterAdmin().prepareGetSnapshots(matchAllPattern()) + .setSnapshots(matchAllPattern()) + .setSort(GetSnapshotsRequest.SortBy.START_TIME) + .setOrder(SortOrder.DESC) + .get() + .getSnapshots(); + assertThat(allSnapshotInfoDesc, is(org.elasticsearch.core.List.of(snapshot3, snapshot2, snapshot1))); + + assertThat(allBeforeStartTimeDescending(startTime3 + 1), is(allSnapshotInfoDesc)); + assertThat(allBeforeStartTimeDescending(startTime3), is(allSnapshotInfoDesc)); + assertThat(allBeforeStartTimeDescending(startTime2), is(org.elasticsearch.core.List.of(snapshot2, snapshot1))); + assertThat(allBeforeStartTimeDescending(startTime1), is(org.elasticsearch.core.List.of(snapshot1))); + assertThat(allBeforeStartTimeDescending(startTime1 - 1), empty()); + } + + // create a snapshot that is guaranteed to have a unique start time + private SnapshotInfo createFullSnapshotWithUniqueStartTime(String repoName, String snapshotName, Set forbiddenStartTimes) { + while (true) { + final SnapshotInfo snapshotInfo = AbstractSnapshotIntegTestCase.createFullSnapshot(logger, repoName, snapshotName); + if (forbiddenStartTimes.contains(snapshotInfo.startTime())) { + logger.info("--> snapshot start time collided"); + assertAcked(clusterAdmin().prepareDeleteSnapshot(repoName, snapshotName).get()); + } else { + assertTrue(forbiddenStartTimes.add(snapshotInfo.startTime())); + return snapshotInfo; + } + } + } + + private List allAfterStartTimeAscending(long timestamp) throws IOException { + final Request request = baseGetSnapshotsRequest("*"); + request.addParameter("sort", GetSnapshotsRequest.SortBy.START_TIME.toString()); + request.addParameter("from_sort_value", String.valueOf(timestamp)); + final Response response = getRestClient().performRequest(request); + return readSnapshotInfos(response).getSnapshots(); + } + + private List allBeforeStartTimeDescending(long timestamp) throws IOException { + final Request request = baseGetSnapshotsRequest("*"); + request.addParameter("sort", GetSnapshotsRequest.SortBy.START_TIME.toString()); + request.addParameter("from_sort_value", String.valueOf(timestamp)); + request.addParameter("order", SortOrder.DESC.toString()); + final Response response = getRestClient().performRequest(request); + return readSnapshotInfos(response).getSnapshots(); + } + private static List getAllSnapshotsForPolicies(String... policies) throws IOException { final Request requestWithPolicy = new Request(HttpGet.METHOD_NAME, "/_snapshot/*/*"); requestWithPolicy.addParameter("slm_policy_filter", Strings.arrayToCommaDelimitedString(policies)); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java index fc00facb3e70e..48fc55a33af48 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java @@ -10,7 +10,6 @@ import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.admin.cluster.repositories.get.TransportGetRepositoriesAction; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequestBuilder; @@ -24,7 +23,9 @@ import java.util.Collection; import java.util.HashSet; import java.util.List; +import java.util.Set; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.in; @@ -389,10 +390,6 @@ public void testNamesStartingInDash() { ); } - private static String[] matchAllPattern() { - return randomBoolean() ? new String[] { "*" } : new String[] { TransportGetRepositoriesAction.ALL_PATTERN }; - } - private List getAllByPatterns(String[] repos, String[] snapshots) { return clusterAdmin().prepareGetSnapshots(repos) .setSnapshots(snapshots) @@ -456,6 +453,185 @@ public void testFilterBySLMPolicy() throws Exception { assertThat(getAllSnapshotsForPolicies(GetSnapshotsRequest.NO_POLICY_PATTERN, "*"), is(allSnapshots)); } + public void testSortAfter() throws Exception { + final String repoName = "test-repo"; + createRepository(repoName, "fs"); + final Set startTimes = new HashSet<>(); + final Set durations = new HashSet<>(); + final SnapshotInfo snapshot1 = createFullSnapshotWithUniqueTimestamps(repoName, "snapshot-1", startTimes, durations); + createIndexWithContent("index-1"); + final SnapshotInfo snapshot2 = createFullSnapshotWithUniqueTimestamps(repoName, "snapshot-2", startTimes, durations); + createIndexWithContent("index-2"); + final SnapshotInfo snapshot3 = createFullSnapshotWithUniqueTimestamps(repoName, "snapshot-3", startTimes, durations); + createIndexWithContent("index-3"); + + final List allSnapshotInfo = clusterAdmin().prepareGetSnapshots(matchAllPattern()) + .setSnapshots(matchAllPattern()) + .setSort(GetSnapshotsRequest.SortBy.START_TIME) + .get() + .getSnapshots(); + assertThat(allSnapshotInfo, is(org.elasticsearch.core.List.of(snapshot1, snapshot2, snapshot3))); + + final long startTime1 = snapshot1.startTime(); + final long startTime2 = snapshot2.startTime(); + final long startTime3 = snapshot3.startTime(); + + assertThat(allAfterStartTimeAscending(startTime1 - 1), is(allSnapshotInfo)); + assertThat(allAfterStartTimeAscending(startTime1), is(allSnapshotInfo)); + assertThat(allAfterStartTimeAscending(startTime2), is(org.elasticsearch.core.List.of(snapshot2, snapshot3))); + assertThat(allAfterStartTimeAscending(startTime3), is(org.elasticsearch.core.List.of(snapshot3))); + assertThat(allAfterStartTimeAscending(startTime3 + 1), empty()); + + final String name1 = snapshot1.snapshotId().getName(); + final String name2 = snapshot2.snapshotId().getName(); + final String name3 = snapshot3.snapshotId().getName(); + + assertThat(allAfterNameAscending("a"), is(allSnapshotInfo)); + assertThat(allAfterNameAscending(name1), is(allSnapshotInfo)); + assertThat(allAfterNameAscending(name2), is(org.elasticsearch.core.List.of(snapshot2, snapshot3))); + assertThat(allAfterNameAscending(name3), is(org.elasticsearch.core.List.of(snapshot3))); + assertThat(allAfterNameAscending("z"), empty()); + + final List allSnapshotInfoDesc = clusterAdmin().prepareGetSnapshots(matchAllPattern()) + .setSnapshots(matchAllPattern()) + .setSort(GetSnapshotsRequest.SortBy.START_TIME) + .setOrder(SortOrder.DESC) + .get() + .getSnapshots(); + assertThat(allSnapshotInfoDesc, is(org.elasticsearch.core.List.of(snapshot3, snapshot2, snapshot1))); + + assertThat(allBeforeStartTimeDescending(startTime3 + 1), is(allSnapshotInfoDesc)); + assertThat(allBeforeStartTimeDescending(startTime3), is(allSnapshotInfoDesc)); + assertThat(allBeforeStartTimeDescending(startTime2), is(org.elasticsearch.core.List.of(snapshot2, snapshot1))); + assertThat(allBeforeStartTimeDescending(startTime1), is(org.elasticsearch.core.List.of(snapshot1))); + assertThat(allBeforeStartTimeDescending(startTime1 - 1), empty()); + + assertThat(allSnapshotInfoDesc, is(org.elasticsearch.core.List.of(snapshot3, snapshot2, snapshot1))); + assertThat(allBeforeNameDescending("z"), is(allSnapshotInfoDesc)); + assertThat(allBeforeNameDescending(name3), is(allSnapshotInfoDesc)); + assertThat(allBeforeNameDescending(name2), is(org.elasticsearch.core.List.of(snapshot2, snapshot1))); + assertThat(allBeforeNameDescending(name1), is(org.elasticsearch.core.List.of(snapshot1))); + assertThat(allBeforeNameDescending("a"), empty()); + + final List allSnapshotInfoByDuration = clusterAdmin().prepareGetSnapshots(matchAllPattern()) + .setSnapshots(matchAllPattern()) + .setSort(GetSnapshotsRequest.SortBy.DURATION) + .get() + .getSnapshots(); + + final long duration1 = allSnapshotInfoByDuration.get(0).endTime() - allSnapshotInfoByDuration.get(0).startTime(); + final long duration2 = allSnapshotInfoByDuration.get(1).endTime() - allSnapshotInfoByDuration.get(1).startTime(); + final long duration3 = allSnapshotInfoByDuration.get(2).endTime() - allSnapshotInfoByDuration.get(2).startTime(); + + assertThat(allAfterDurationAscending(duration1 - 1), is(allSnapshotInfoByDuration)); + assertThat(allAfterDurationAscending(duration1), is(allSnapshotInfoByDuration)); + assertThat(allAfterDurationAscending(duration2), is(allSnapshotInfoByDuration.subList(1, 3))); + assertThat(allAfterDurationAscending(duration3), is(org.elasticsearch.core.List.of(allSnapshotInfoByDuration.get(2)))); + assertThat(allAfterDurationAscending(duration3 + 1), empty()); + + final List allSnapshotInfoByDurationDesc = clusterAdmin().prepareGetSnapshots(matchAllPattern()) + .setSnapshots(matchAllPattern()) + .setSort(GetSnapshotsRequest.SortBy.DURATION) + .setOrder(SortOrder.DESC) + .get() + .getSnapshots(); + + assertThat(allBeforeDurationDescending(duration3 + 1), is(allSnapshotInfoByDurationDesc)); + assertThat(allBeforeDurationDescending(duration3), is(allSnapshotInfoByDurationDesc)); + assertThat(allBeforeDurationDescending(duration2), is(allSnapshotInfoByDurationDesc.subList(1, 3))); + assertThat(allBeforeDurationDescending(duration1), is(org.elasticsearch.core.List.of(allSnapshotInfoByDurationDesc.get(2)))); + assertThat(allBeforeDurationDescending(duration1 - 1), empty()); + + final SnapshotInfo otherSnapshot = createFullSnapshot(repoName, "other-snapshot"); + + assertThat(allSnapshots(new String[] { "snap*" }, GetSnapshotsRequest.SortBy.NAME, SortOrder.ASC, "a"), is(allSnapshotInfo)); + assertThat( + allSnapshots(new String[] { "o*" }, GetSnapshotsRequest.SortBy.NAME, SortOrder.ASC, "a"), + is(org.elasticsearch.core.List.of(otherSnapshot)) + ); + + final GetSnapshotsResponse paginatedResponse = clusterAdmin().prepareGetSnapshots(matchAllPattern()) + .setSnapshots("snap*") + .setSort(GetSnapshotsRequest.SortBy.NAME) + .setFromSortValue("a") + .setOffset(1) + .setSize(1) + .get(); + assertThat(paginatedResponse.getSnapshots(), is(org.elasticsearch.core.List.of(snapshot2))); + assertThat(paginatedResponse.totalCount(), is(3)); + final GetSnapshotsResponse paginatedResponse2 = clusterAdmin().prepareGetSnapshots(matchAllPattern()) + .setSnapshots("snap*") + .setSort(GetSnapshotsRequest.SortBy.NAME) + .setFromSortValue("a") + .setOffset(0) + .setSize(2) + .get(); + assertThat(paginatedResponse2.getSnapshots(), is(org.elasticsearch.core.List.of(snapshot1, snapshot2))); + assertThat(paginatedResponse2.totalCount(), is(3)); + } + + // Create a snapshot that is guaranteed to have a unique start time and duration for tests around ordering by either. + // Don't use this with more than 3 snapshots on platforms with low-resolution clocks as the durations could always collide there + // causing an infinite loop + private SnapshotInfo createFullSnapshotWithUniqueTimestamps( + String repoName, + String snapshotName, + Set forbiddenStartTimes, + Set forbiddenDurations + ) throws Exception { + while (true) { + final SnapshotInfo snapshotInfo = createFullSnapshot(repoName, snapshotName); + final long duration = snapshotInfo.endTime() - snapshotInfo.startTime(); + if (forbiddenStartTimes.contains(snapshotInfo.startTime()) || forbiddenDurations.contains(duration)) { + logger.info("--> snapshot start time or duration collided"); + assertAcked(startDeleteSnapshot(repoName, snapshotName).get()); + } else { + assertTrue(forbiddenStartTimes.add(snapshotInfo.startTime())); + assertTrue(forbiddenDurations.add(duration)); + return snapshotInfo; + } + } + } + + private List allAfterStartTimeAscending(long timestamp) { + return allSnapshots(matchAllPattern(), GetSnapshotsRequest.SortBy.START_TIME, SortOrder.ASC, timestamp); + } + + private List allBeforeStartTimeDescending(long timestamp) { + return allSnapshots(matchAllPattern(), GetSnapshotsRequest.SortBy.START_TIME, SortOrder.DESC, timestamp); + } + + private List allAfterNameAscending(String name) { + return allSnapshots(matchAllPattern(), GetSnapshotsRequest.SortBy.NAME, SortOrder.ASC, name); + } + + private List allBeforeNameDescending(String name) { + return allSnapshots(matchAllPattern(), GetSnapshotsRequest.SortBy.NAME, SortOrder.DESC, name); + } + + private List allAfterDurationAscending(long duration) { + return allSnapshots(matchAllPattern(), GetSnapshotsRequest.SortBy.DURATION, SortOrder.ASC, duration); + } + + private List allBeforeDurationDescending(long duration) { + return allSnapshots(matchAllPattern(), GetSnapshotsRequest.SortBy.DURATION, SortOrder.DESC, duration); + } + + private static List allSnapshots( + String[] snapshotNames, + GetSnapshotsRequest.SortBy sortBy, + SortOrder order, + Object fromSortValue + ) { + return clusterAdmin().prepareGetSnapshots(matchAllPattern()) + .setSnapshots(snapshotNames) + .setSort(sortBy) + .setFromSortValue(fromSortValue.toString()) + .setOrder(order) + .get() + .getSnapshots(); + } + private static List getAllSnapshotsForPolicies(String... policies) { return clusterAdmin().prepareGetSnapshots(matchAllPattern()) .setSnapshots(matchAllPattern()) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java index 46581ed3fd2cb..3a9a2c23d8034 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java @@ -42,6 +42,8 @@ public class GetSnapshotsRequest extends MasterNodeRequest public static final Version SLM_POLICY_FILTERING_VERSION = Version.V_7_16_0; + public static final Version FROM_SORT_VALUE_VERSION = Version.V_7_16_0; + public static final Version MULTIPLE_REPOSITORIES_SUPPORT_ADDED = Version.V_7_14_0; public static final Version PAGINATED_GET_SNAPSHOTS_VERSION = Version.V_7_14_0; @@ -65,6 +67,9 @@ public class GetSnapshotsRequest extends MasterNodeRequest @Nullable private After after; + @Nullable + private String fromSortValue; + private SortBy sort = SortBy.START_TIME; private SortOrder order = SortOrder.ASC; @@ -133,6 +138,9 @@ public GetSnapshotsRequest(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(SLM_POLICY_FILTERING_VERSION)) { policies = in.readStringArray(); } + if (in.getVersion().onOrAfter(FROM_SORT_VALUE_VERSION)) { + fromSortValue = in.readOptionalString(); + } } } @@ -182,6 +190,11 @@ public void writeTo(StreamOutput out) throws IOException { "can't use slm policy filter in snapshots request with node version [" + out.getVersion() + "]" ); } + if (out.getVersion().onOrAfter(FROM_SORT_VALUE_VERSION)) { + out.writeOptionalString(fromSortValue); + } else if (fromSortValue != null) { + throw new IllegalArgumentException("can't use after-value in snapshot request with node version [" + out.getVersion() + "]"); + } } @Override @@ -212,8 +225,15 @@ public ActionRequestValidationException validate() { if (policies.length != 0) { validationException = addValidationError("can't use slm policy filter with verbose=false", validationException); } - } else if (after != null && offset > 0) { - validationException = addValidationError("can't use after and offset simultaneously", validationException); + if (fromSortValue != null) { + validationException = addValidationError("can't use from_sort_value with verbose=false", validationException); + } + } else if (offset > 0) { + if (after != null) { + validationException = addValidationError("can't use after and offset simultaneously", validationException); + } + } else if (after != null && fromSortValue != null) { + validationException = addValidationError("can't use after and from_sort_value simultaneously", validationException); } return validationException; } @@ -353,6 +373,16 @@ public GetSnapshotsRequest after(@Nullable After after) { return this; } + public GetSnapshotsRequest fromSortValue(@Nullable String fromSortValue) { + this.fromSortValue = fromSortValue; + return this; + } + + @Nullable + public String fromSortValue() { + return fromSortValue; + } + public GetSnapshotsRequest sort(SortBy sort) { this.sort = sort; return this; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java index 8c43d3e2d961c..49902f9622591 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java @@ -142,6 +142,11 @@ public GetSnapshotsRequestBuilder setAfter(@Nullable GetSnapshotsRequest.After a return this; } + public GetSnapshotsRequestBuilder setFromSortValue(@Nullable String fromSortValue) { + request.fromSortValue(fromSortValue); + return this; + } + public GetSnapshotsRequestBuilder setSort(GetSnapshotsRequest.SortBy sort) { request.sort(sort); return this; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index b2649a5575b6b..80203ef59095c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -123,7 +123,7 @@ protected void masterOperation( request.offset(), request.size(), request.order(), - request.policies(), + buildSnapshotPredicate(request.sort(), request.order(), request.policies(), request.fromSortValue()), listener ); } @@ -141,7 +141,7 @@ private void getMultipleReposSnapshotInfo( int offset, int size, SortOrder order, - String[] slmPolicies, + @Nullable Predicate predicate, ActionListener listener ) { // short-circuit if there are no repos, because we can not create GroupedActionListener of size 0 @@ -161,7 +161,7 @@ private void getMultipleReposSnapshotInfo( .map(Tuple::v1) .filter(Objects::nonNull) .collect(Collectors.toMap(Tuple::v1, Tuple::v2)); - final SnapshotsInRepo snInfos = sortAndFilterSnapshots(allSnapshots, sortBy, after, offset, size, order, slmPolicies); + final SnapshotsInRepo snInfos = sortAndFilterSnapshots(allSnapshots, sortBy, after, offset, size, order, predicate); final List snapshotInfos = snInfos.snapshotInfos; final int remaining = snInfos.remaining + responses.stream() .map(Tuple::v2) @@ -185,7 +185,7 @@ private void getMultipleReposSnapshotInfo( snapshotsInProgress, repoName, snapshots, - slmPolicies, + predicate, ignoreUnavailable, verbose, cancellableTask, @@ -207,7 +207,7 @@ private void getSingleRepoSnapshotInfo( SnapshotsInProgress snapshotsInProgress, String repo, String[] snapshots, - String[] slmPolicies, + Predicate predicate, boolean ignoreUnavailable, boolean verbose, CancellableTask task, @@ -245,7 +245,7 @@ private void getSingleRepoSnapshotInfo( sortBy, after, order, - slmPolicies, + predicate, listener ), listener::onFailure @@ -285,7 +285,7 @@ private void loadSnapshotInfos( GetSnapshotsRequest.SortBy sortBy, @Nullable final GetSnapshotsRequest.After after, SortOrder order, - String[] slmPolicies, + @Nullable Predicate predicate, ActionListener listener ) { if (task.notifyIfCancelled(listener)) { @@ -357,14 +357,11 @@ private void loadSnapshotInfos( sortBy, after, order, - slmPolicies, + predicate, listener ); } else { - assert slmPolicies.length == 0 - : "slm policy filtering not support for non-verbose request but saw [" - + Strings.arrayToCommaDelimitedString(slmPolicies) - + "]"; + assert predicate == null : "filtering is not supported in non-verbose mode"; final SnapshotsInRepo snapshotInfos; if (repositoryData != null) { // want non-current snapshots as well, which are found in the repository data @@ -400,7 +397,7 @@ private void snapshots( GetSnapshotsRequest.SortBy sortBy, @Nullable GetSnapshotsRequest.After after, SortOrder order, - String[] slmPolicies, + @Nullable Predicate predicate, ActionListener listener ) { if (task.notifyIfCancelled(listener)) { @@ -429,7 +426,7 @@ private void snapshots( final ActionListener allDoneListener = listener.delegateFailure((l, v) -> { final ArrayList snapshotList = new ArrayList<>(snapshotInfos); snapshotList.addAll(snapshotSet); - listener.onResponse(sortAndFilterSnapshots(snapshotList, sortBy, after, 0, GetSnapshotsRequest.NO_LIMIT, order, slmPolicies)); + listener.onResponse(sortAndFilterSnapshots(snapshotList, sortBy, after, 0, GetSnapshotsRequest.NO_LIMIT, order, predicate)); }); if (snapshotIdsToIterate.isEmpty()) { allDoneListener.onResponse(null); @@ -524,17 +521,38 @@ private static SnapshotsInRepo sortAndFilterSnapshots( final int offset, final int size, final SortOrder order, - final String[] slmPolicies + final @Nullable Predicate predicate ) { final List filteredSnapshotInfos; - if (slmPolicies.length == 0) { + if (predicate == null) { filteredSnapshotInfos = snapshotInfos; } else { - filteredSnapshotInfos = filterBySLMPolicies(snapshotInfos, slmPolicies); + filteredSnapshotInfos = Collections.unmodifiableList(snapshotInfos.stream().filter(predicate).collect(Collectors.toList())); } return sortSnapshots(filteredSnapshotInfos, sortBy, after, offset, size, order); } + private static Predicate buildSnapshotPredicate( + GetSnapshotsRequest.SortBy sortBy, + SortOrder order, + String[] slmPolicies, + String fromSortValue + ) { + Predicate predicate = null; + if (slmPolicies.length > 0) { + predicate = filterBySLMPolicies(slmPolicies); + } + if (fromSortValue != null) { + final Predicate fromSortValuePredicate = buildFromSortValuePredicate(sortBy, fromSortValue, order, null, null); + if (predicate == null) { + predicate = fromSortValuePredicate; + } else { + predicate = fromSortValuePredicate.and(predicate); + } + } + return predicate; + } + private static SnapshotsInRepo sortSnapshots( List snapshotInfos, GetSnapshotsRequest.SortBy sortBy, @@ -574,57 +592,7 @@ private static SnapshotsInRepo sortSnapshots( if (after != null) { assert offset == 0 : "can't combine after and offset but saw [" + after + "] and offset [" + offset + "]"; - final Predicate isAfter; - final String snapshotName = after.snapshotName(); - final String repoName = after.repoName(); - switch (sortBy) { - case START_TIME: - isAfter = filterByLongOffset(SnapshotInfo::startTime, Long.parseLong(after.value()), snapshotName, repoName, order); - break; - case NAME: - isAfter = order == SortOrder.ASC - ? (info -> compareName(snapshotName, repoName, info) < 0) - : (info -> compareName(snapshotName, repoName, info) > 0); - break; - case DURATION: - isAfter = filterByLongOffset( - info -> info.endTime() - info.startTime(), - Long.parseLong(after.value()), - snapshotName, - repoName, - order - ); - break; - case INDICES: - isAfter = filterByLongOffset( - info -> info.indices().size(), - Integer.parseInt(after.value()), - snapshotName, - repoName, - order - ); - break; - case SHARDS: - isAfter = filterByLongOffset(SnapshotInfo::totalShards, Integer.parseInt(after.value()), snapshotName, repoName, order); - break; - case FAILED_SHARDS: - isAfter = filterByLongOffset( - SnapshotInfo::failedShards, - Integer.parseInt(after.value()), - snapshotName, - repoName, - order - ); - break; - case REPOSITORY: - isAfter = order == SortOrder.ASC - ? (info -> compareRepositoryName(snapshotName, repoName, info) < 0) - : (info -> compareRepositoryName(snapshotName, repoName, info) > 0); - break; - default: - throw new AssertionError("unexpected sort column [" + sortBy + "]"); - } - infos = infos.filter(isAfter); + infos = infos.filter(buildFromSortValuePredicate(sortBy, after.value(), order, after.snapshotName(), after.repoName())); } infos = infos.sorted(order == SortOrder.DESC ? comparator.reversed() : comparator).skip(offset); final List allSnapshots = infos.collect(Collectors.toList()); @@ -640,7 +608,67 @@ private static SnapshotsInRepo sortSnapshots( return new SnapshotsInRepo(resultSet, snapshotInfos.size(), allSnapshots.size() - resultSet.size()); } - private static List filterBySLMPolicies(List snapshotInfos, String[] slmPolicies) { + private static Predicate buildFromSortValuePredicate( + GetSnapshotsRequest.SortBy sortBy, + String after, + SortOrder order, + @Nullable String snapshotName, + @Nullable String repoName + ) { + final Predicate isAfter; + switch (sortBy) { + case START_TIME: + isAfter = filterByLongOffset(SnapshotInfo::startTime, Long.parseLong(after), snapshotName, repoName, order); + break; + case NAME: + if (snapshotName == null) { + assert repoName == null : "no snapshot name given but saw repo name [" + repoName + "]"; + isAfter = order == SortOrder.ASC + ? snapshotInfo -> after.compareTo(snapshotInfo.snapshotId().getName()) <= 0 + : snapshotInfo -> after.compareTo(snapshotInfo.snapshotId().getName()) >= 0; + } else { + isAfter = order == SortOrder.ASC + ? (info -> compareName(snapshotName, repoName, info) < 0) + : (info -> compareName(snapshotName, repoName, info) > 0); + } + break; + case DURATION: + isAfter = filterByLongOffset( + info -> info.endTime() - info.startTime(), + Long.parseLong(after), + snapshotName, + repoName, + order + ); + break; + case INDICES: + isAfter = filterByLongOffset(info -> info.indices().size(), Integer.parseInt(after), snapshotName, repoName, order); + break; + case SHARDS: + isAfter = filterByLongOffset(SnapshotInfo::totalShards, Integer.parseInt(after), snapshotName, repoName, order); + break; + case FAILED_SHARDS: + isAfter = filterByLongOffset(SnapshotInfo::failedShards, Integer.parseInt(after), snapshotName, repoName, order); + break; + case REPOSITORY: + if (snapshotName == null) { + assert repoName == null : "no snapshot name given but saw repo name [" + repoName + "]"; + isAfter = order == SortOrder.ASC + ? snapshotInfo -> after.compareTo(snapshotInfo.repository()) <= 0 + : snapshotInfo -> after.compareTo(snapshotInfo.repository()) >= 0; + } else { + isAfter = order == SortOrder.ASC + ? (info -> compareRepositoryName(snapshotName, repoName, info) < 0) + : (info -> compareRepositoryName(snapshotName, repoName, info) > 0); + } + break; + default: + throw new AssertionError("unexpected sort column [" + sortBy + "]"); + } + return isAfter; + } + + private static Predicate filterBySLMPolicies(String[] slmPolicies) { final List includePatterns = new ArrayList<>(); final List excludePatterns = new ArrayList<>(); boolean seenWildcard = false; @@ -660,7 +688,7 @@ private static List filterBySLMPolicies(List snapsho final String[] includes = includePatterns.toArray(Strings.EMPTY_ARRAY); final String[] excludes = excludePatterns.toArray(Strings.EMPTY_ARRAY); final boolean matchWithoutPolicy = matchNoPolicy; - return Collections.unmodifiableList(snapshotInfos.stream().filter(snapshotInfo -> { + return snapshotInfo -> { final Map metadata = snapshotInfo.userMetadata(); final String policy; if (metadata == null) { @@ -676,16 +704,20 @@ private static List filterBySLMPolicies(List snapsho return false; } return excludes.length == 0 || Regex.simpleMatch(excludes, policy) == false; - }).collect(Collectors.toList())); + }; } private static Predicate filterByLongOffset( ToLongFunction extractor, long after, - String snapshotName, - String repoName, + @Nullable String snapshotName, + @Nullable String repoName, SortOrder order ) { + if (snapshotName == null) { + assert repoName == null : "no snapshot name given but saw repo name [" + repoName + "]"; + return order == SortOrder.ASC ? info -> after <= extractor.applyAsLong(info) : info -> after >= extractor.applyAsLong(info); + } return order == SortOrder.ASC ? info -> { final long val = extractor.applyAsLong(info); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetSnapshotsAction.java index 732ae489c41e9..0f60282111d00 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetSnapshotsAction.java @@ -65,6 +65,10 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC if (afterString != null) { getSnapshotsRequest.after(GetSnapshotsRequest.After.fromQueryParam(afterString)); } + final String fromSortValue = request.param("from_sort_value"); + if (fromSortValue != null) { + getSnapshotsRequest.fromSortValue(fromSortValue); + } final String[] policies = request.paramAsStringArray("slm_policy_filter", Strings.EMPTY_ARRAY); getSnapshotsRequest.policies(policies); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestTests.java index 589043a5a6565..965654266e1cc 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestTests.java @@ -55,6 +55,11 @@ public void testValidateParameters() { final ActionRequestValidationException e = request.validate(); assertThat(e.getMessage(), containsString("can't use after with verbose=false")); } + { + final GetSnapshotsRequest request = new GetSnapshotsRequest("repo", "snapshot").verbose(false).fromSortValue("bar"); + final ActionRequestValidationException e = request.validate(); + assertThat(e.getMessage(), containsString("can't use from_sort_value with verbose=false")); + } { final GetSnapshotsRequest request = new GetSnapshotsRequest("repo", "snapshot").after( new GetSnapshotsRequest.After("foo", "repo", "bar") @@ -62,6 +67,12 @@ public void testValidateParameters() { final ActionRequestValidationException e = request.validate(); assertThat(e.getMessage(), containsString("can't use after and offset simultaneously")); } + { + final GetSnapshotsRequest request = new GetSnapshotsRequest("repo", "snapshot").fromSortValue("foo") + .after(new GetSnapshotsRequest.After("foo", "repo", "bar")); + final ActionRequestValidationException e = request.validate(); + assertThat(e.getMessage(), containsString("can't use after and from_sort_value simultaneously")); + } { final GetSnapshotsRequest request = new GetSnapshotsRequest("repo", "snapshot").policies("some-policy").verbose(false); final ActionRequestValidationException e = request.validate(); diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java index d4884c725406d..bac671a4957ba 100644 --- a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; +import org.elasticsearch.action.admin.cluster.repositories.get.TransportGetRepositoriesAction; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; import org.elasticsearch.action.index.IndexRequestBuilder; @@ -405,6 +406,10 @@ protected String initWithSnapshotVersion(String repoName, Path repoPath, Version } protected SnapshotInfo createFullSnapshot(String repoName, String snapshotName) { + return createFullSnapshot(logger, repoName, snapshotName); + } + + public static SnapshotInfo createFullSnapshot(Logger logger, String repoName, String snapshotName) { logger.info("--> creating full snapshot [{}] in [{}]", snapshotName, repoName); CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(repoName, snapshotName) .setIncludeGlobalState(true) @@ -760,4 +765,8 @@ public static Map randomUserMetadata() { } return metadata; } + + public static String[] matchAllPattern() { + return randomBoolean() ? new String[] { "*" } : new String[] { TransportGetRepositoriesAction.ALL_PATTERN }; + } }