Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Get snapshots support for multiple repositories #42090

Merged
merged 40 commits into from
Jun 19, 2019
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
80edff1
Get snapshots support for multiple repositories
May 10, 2019
c2e24f2
Run on GENERIC thread pool
May 23, 2019
94e60dd
Use GroupedActionListener
May 23, 2019
3bbea57
Use fs repository
May 23, 2019
94f4d63
Remove supplier
May 23, 2019
845ed5f
Add assertions
May 23, 2019
c00640f
Use GetSnapshotsResponse.Response in GroupedActionListener
May 23, 2019
3963638
Use unmodifiableMap in the constructor
May 23, 2019
e8ec22c
Checkstyle
May 23, 2019
0e956ae
Moar checkstyle
May 24, 2019
71f6aa1
BWC
May 24, 2019
adaa376
Fix indentation
May 24, 2019
2a49439
Merge branch 'master' into multi_repos
May 24, 2019
85b2953
Add docs
May 24, 2019
5fa3d16
Fix docs
May 24, 2019
3a8743f
Fix FullClusterRestartIT
May 24, 2019
ec8bce7
Fix tests
May 28, 2019
12a563d
Checkstyle
May 28, 2019
62a7acc
Moar fixes
May 28, 2019
94d3a5b
Fix
May 28, 2019
068324a
Merge branch 'master' into multi_repos
May 28, 2019
cf0bbda
Fix merge
May 28, 2019
04d494b
Merge branch 'master' into multi_repos
May 31, 2019
ff4c89b
More fixes
May 31, 2019
16c69ac
Fix yml
May 31, 2019
265f18e
s3 yml fixes
May 31, 2019
be898f4
Revert reformatting
Jun 3, 2019
33cad5e
Use ActionRunnable
Jun 3, 2019
ca011e3
buildSimpleSnapshotInfos
Jun 3, 2019
752fbc8
Merge branch 'master' into multi_repos
Jun 14, 2019
84430b2
Fix typo lost during force push
Jun 14, 2019
6582508
Change cat shapshots docs
Jun 14, 2019
35a171f
\n
Jun 14, 2019
dc64cea
Fix "Get snapshot info with metadata"
Jun 14, 2019
682d583
readFrom should throw UnsupportedOpExc
Jun 17, 2019
f19ff8f
Add exceptions as cause in cat API
Jun 17, 2019
e2f2e9a
Make repositories final
Jun 17, 2019
e441032
Fix GetRepositoriesResponse streamable and get rid of rid of readFrom
Jun 17, 2019
c6435a1
Fix HDFS yml tests
Jun 17, 2019
77a7328
Merge branch 'master' into multi_repos
Jun 18, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ static Request createSnapshot(CreateSnapshotRequest createSnapshotRequest) throw

static Request getSnapshots(GetSnapshotsRequest getSnapshotsRequest) {
RequestConverters.EndpointBuilder endpointBuilder = new RequestConverters.EndpointBuilder().addPathPartAsIs("_snapshot")
.addPathPart(getSnapshotsRequest.repository());
.addCommaSeparatedPathParts(getSnapshotsRequest.repositories());
String endpoint;
if (getSnapshotsRequest.snapshots().length == 0) {
endpoint = endpointBuilder.addPathPart("_all").build();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,14 @@
import org.elasticsearch.repositories.fs.FsRepository;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.snapshots.RestoreInfo;
import org.mockito.internal.util.collections.Sets;

import java.io.IOException;
import java.util.Collections;
import java.util.stream.Collectors;

import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is;

public class SnapshotIT extends ESRestHighLevelClientTestCase {
Expand Down Expand Up @@ -155,39 +155,48 @@ public void testCreateSnapshot() throws IOException {
}

public void testGetSnapshots() throws IOException {
String repository = "test_repository";
String repository1 = "test_repository1";
String repository2 = "test_repository2";
String snapshot1 = "test_snapshot1";
String snapshot2 = "test_snapshot2";

AcknowledgedResponse putRepositoryResponse = createTestRepository(repository, FsRepository.TYPE, "{\"location\": \".\"}");
AcknowledgedResponse putRepositoryResponse =
createTestRepository(repository1, FsRepository.TYPE, "{\"location\": \"loc1\"}");
assertTrue(putRepositoryResponse.isAcknowledged());

CreateSnapshotRequest createSnapshotRequest1 = new CreateSnapshotRequest(repository, snapshot1);
AcknowledgedResponse putRepositoryResponse2 =
createTestRepository(repository2, FsRepository.TYPE, "{\"location\": \"loc2\"}");
assertTrue(putRepositoryResponse2.isAcknowledged());

CreateSnapshotRequest createSnapshotRequest1 = new CreateSnapshotRequest(repository1, snapshot1);
createSnapshotRequest1.waitForCompletion(true);
CreateSnapshotResponse putSnapshotResponse1 = createTestSnapshot(createSnapshotRequest1);
CreateSnapshotRequest createSnapshotRequest2 = new CreateSnapshotRequest(repository, snapshot2);
CreateSnapshotRequest createSnapshotRequest2 = new CreateSnapshotRequest(repository2, snapshot2);
createSnapshotRequest2.waitForCompletion(true);
CreateSnapshotResponse putSnapshotResponse2 = createTestSnapshot(createSnapshotRequest2);
// check that the request went ok without parsing JSON here. When using the high level client, check acknowledgement instead.
assertEquals(RestStatus.OK, putSnapshotResponse1.status());
assertEquals(RestStatus.OK, putSnapshotResponse2.status());

GetSnapshotsRequest request;
if (randomBoolean()) {
request = new GetSnapshotsRequest(repository);
} else if (randomBoolean()) {
request = new GetSnapshotsRequest(repository, new String[] {"_all"});
GetSnapshotsRequest request = new GetSnapshotsRequest(
randomFrom(new String[]{"_all"}, new String[]{"*"}, new String[]{repository1, repository2}),
randomFrom(new String[]{"_all"}, new String[]{"*"}, new String[]{snapshot1, snapshot2})
);
request.ignoreUnavailable(true);

} else {
request = new GetSnapshotsRequest(repository, new String[] {snapshot1, snapshot2});
}
GetSnapshotsResponse response = execute(request, highLevelClient().snapshot()::get, highLevelClient().snapshot()::getAsync);

assertEquals(2, response.getSnapshots().size());
assertThat(response.getSnapshots().stream().map((s) -> s.snapshotId().getName()).collect(Collectors.toList()),
contains("test_snapshot1", "test_snapshot2"));
assertThat(response.isFailed(), is(false));
assertThat(response.getRepositories(), equalTo(Sets.newSet(repository1, repository2)));

assertThat(response.getSnapshots(repository1), hasSize(1));
assertThat(response.getSnapshots(repository1).get(0).snapshotId().getName(), equalTo(snapshot1));

assertThat(response.getSnapshots(repository2), hasSize(1));
assertThat(response.getSnapshots(repository2).get(0).snapshotId().getName(), equalTo(snapshot2));
}


public void testSnapshotsStatus() throws IOException {
String testRepository = "test";
String testSnapshot = "snapshot";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@

import java.io.IOException;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
Expand Down Expand Up @@ -148,15 +147,16 @@ public void testCreateSnapshot() throws IOException {

public void testGetSnapshots() {
Map<String, String> expectedParams = new HashMap<>();
String repository = RequestConvertersTests.randomIndicesNames(1, 1)[0];
String repository1 = randomAlphaOfLength(10);
String repository2 = randomAlphaOfLength(10);
String snapshot1 = "snapshot1-" + randomAlphaOfLengthBetween(2, 5).toLowerCase(Locale.ROOT);
String snapshot2 = "snapshot2-" + randomAlphaOfLengthBetween(2, 5).toLowerCase(Locale.ROOT);

String endpoint = String.format(Locale.ROOT, "/_snapshot/%s/%s,%s", repository, snapshot1, snapshot2);
String endpoint = String.format(Locale.ROOT, "/_snapshot/%s,%s/%s,%s", repository1, repository2, snapshot1, snapshot2);

GetSnapshotsRequest getSnapshotsRequest = new GetSnapshotsRequest();
getSnapshotsRequest.repository(repository);
getSnapshotsRequest.snapshots(Arrays.asList(snapshot1, snapshot2).toArray(new String[0]));
getSnapshotsRequest.repositories(repository1, repository2);
getSnapshotsRequest.snapshots(new String[]{snapshot1, snapshot2});
RequestConvertersTests.setRandomMasterTimeout(getSnapshotsRequest, expectedParams);

if (randomBoolean()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -590,7 +590,7 @@ public void testSnapshotGetSnapshots() throws IOException {
// end::get-snapshots-request

// tag::get-snapshots-request-repositoryName
request.repository(repositoryName); // <1>
request.repositories(repositoryName); // <1>
// end::get-snapshots-request-repositoryName

// tag::get-snapshots-request-snapshots
Expand All @@ -616,7 +616,7 @@ public void testSnapshotGetSnapshots() throws IOException {
// end::get-snapshots-execute

// tag::get-snapshots-response
List<SnapshotInfo> snapshotsInfos = response.getSnapshots();
List<SnapshotInfo> snapshotsInfos = response.getSnapshots(repositoryName);
SnapshotInfo snapshotInfo = snapshotsInfos.get(0);
RestStatus restStatus = snapshotInfo.status(); // <1>
SnapshotId snapshotId = snapshotInfo.snapshotId(); // <2>
Expand Down
6 changes: 3 additions & 3 deletions docs/reference/cat/snapshots.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ Which looks like:

[source,txt]
--------------------------------------------------
id status start_epoch start_time end_epoch end_time duration indices successful_shards failed_shards total_shards
snap1 FAILED 1445616705 18:11:45 1445616978 18:16:18 4.6m 1 4 1 5
snap2 SUCCESS 1445634298 23:04:58 1445634672 23:11:12 6.2m 2 10 0 10
id repository status start_epoch start_time end_epoch end_time duration indices successful_shards failed_shards total_shards
snap1 repo1 FAILED 1445616705 18:11:45 1445616978 18:16:18 4.6m 1 4 1 5
snap2 repo1 SUCCESS 1445634298 23:04:58 1445634672 23:11:12 6.2m 2 10 0 10
--------------------------------------------------
// TESTRESPONSE[s/FAILED/SUCCESS/ s/14456\d+/\\d+/ s/\d+(\.\d+)?(m|s|ms)/\\d+(\\.\\d+)?(m|s|ms)/]
// TESTRESPONSE[s/\d+:\d+:\d+/\\d+:\\d+:\\d+/]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ public void testUrlRepository() throws Exception {
.prepareGetSnapshots("test-repo")
.setSnapshots("test-snap")
.get()
.getSnapshots()
.getSnapshots("test-repo")
.get(0)
.state();
assertThat(state, equalTo(SnapshotState.SUCCESS));
Expand Down Expand Up @@ -116,16 +116,16 @@ public void testUrlRepository() throws Exception {

logger.info("--> list available shapshots");
GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get();
assertThat(getSnapshotsResponse.getSnapshots(), notNullValue());
assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1));
assertThat(getSnapshotsResponse.getSnapshots("url-repo"), notNullValue());
assertThat(getSnapshotsResponse.getSnapshots("url-repo").size(), equalTo(1));

logger.info("--> delete snapshot");
AcknowledgedResponse deleteSnapshotResponse = client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap").get();
assertAcked(deleteSnapshotResponse);

logger.info("--> list available shapshot again, no snapshots should be returned");
getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get();
assertThat(getSnapshotsResponse.getSnapshots(), notNullValue());
assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(0));
assertThat(getSnapshotsResponse.getSnapshots("url-repo"), notNullValue());
assertThat(getSnapshotsResponse.getSnapshots("url-repo").size(), equalTo(0));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ public void testSimpleWorkflow() {
.prepareGetSnapshots("test-repo")
.setSnapshots("test-snap")
.get()
.getSnapshots()
.getSnapshots("test-repo")
.get(0)
.state(),
equalTo(SnapshotState.SUCCESS));
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,17 @@
---
"Help":
- skip:
version: " - 7.9.99"
reason: Repository field added in 8.0

- do:
cat.snapshots:
help: true

- match:
$body: |
/^ id .+ \n
repository .+ \n
status .+ \n
start_epoch .+ \n
start_time .+ \n
Expand All @@ -21,6 +26,9 @@
$/
---
"Test cat snapshots output":
- skip:
version: " - 7.9.99"
reason: Repository field added in 8.0

- do:
snapshot.create_repository:
Expand Down Expand Up @@ -74,6 +82,6 @@

- match:
$body: |
/^ snap1\s+ SUCCESS\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \S+\s+ 2\s+ 2\s+ 0\s+ 2\s*\n
snap2\s+ SUCCESS\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \S+\s+ 2\s+ 2\s+ 0\s+ 2\s*\n
/^ snap1\s+ test_cat_snapshots_1\+ SUCCESS\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \S+\s+ 2\s+ 2\s+ 0\s+ 2\s*\n
snap2\s+ test_cat_snapshots_1\+ SUCCESS\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \S+\s+ 2\s+ 2\s+ 0\s+ 2\s*\n
$/
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,10 @@ public class GetRepositoriesResponse extends ActionResponse implements ToXConten
this.repositories = repositories;
}

public GetRepositoriesResponse(StreamInput in) throws IOException {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why add this but not fully convert class (see e.g. AddVotingConfigExclusionsResponse)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure I understand, could you please clarify?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We generally want to move away from the Streamable interface and retire its readFrom(StreamInput) method. We have been doing this step-by-step throughout the codebase. The replacement is to define a constructor on the request/response object which takes StreamInput as parameter. This also allows us to make many fields in these (Request/Reponse) classes final (see e.g. GetCcrRestoreFileChunkRequest). The PR here does half of the work in this regard. It adds a constructor that just delegates to the readFrom method, i.e. adding more code without the benefit of decluttering the existing one.

Copy link
Contributor Author

@andrershov andrershov Jun 17, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ywelsch thanks for the clarification, 682d583 and e2f2e9a

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ywelsch sorry, I made it wrong first time and fixed here e441032. I have also made similar changes to GetSnapshotsResponse as a part of this commit.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

readFrom(in);
}

/**
* List of repositories to return
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.elasticsearch.action.admin.cluster.snapshots.get;

import org.elasticsearch.Version;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.support.master.MasterNodeRequest;
import org.elasticsearch.common.Strings;
Expand All @@ -37,8 +38,9 @@ public class GetSnapshotsRequest extends MasterNodeRequest<GetSnapshotsRequest>
public static final String ALL_SNAPSHOTS = "_all";
public static final String CURRENT_SNAPSHOT = "_current";
public static final boolean DEFAULT_VERBOSE_MODE = true;
public static final Version MULTIPLE_REPOSITORIES_SUPPORT_ADDED = Version.V_8_0_0;

private String repository;
private String[] repositories;

private String[] snapshots = Strings.EMPTY_ARRAY;

Expand All @@ -50,28 +52,32 @@ public GetSnapshotsRequest() {
}

/**
* Constructs a new get snapshots request with given repository name and list of snapshots
* Constructs a new get snapshots request with given repository names and list of snapshots
*
* @param repository repository name
* @param repositories repository names
* @param snapshots list of snapshots
*/
public GetSnapshotsRequest(String repository, String[] snapshots) {
this.repository = repository;
public GetSnapshotsRequest(String[] repositories, String[] snapshots) {
this.repositories = repositories;
this.snapshots = snapshots;
}

/**
* Constructs a new get snapshots request with given repository name
* Constructs a new get snapshots request with given repository names
*
* @param repository repository name
* @param repositories repository names
*/
public GetSnapshotsRequest(String repository) {
this.repository = repository;
public GetSnapshotsRequest(String... repositories) {
this.repositories = repositories;
}

public GetSnapshotsRequest(StreamInput in) throws IOException {
super(in);
repository = in.readString();
if (in.getVersion().onOrAfter(MULTIPLE_REPOSITORIES_SUPPORT_ADDED)) {
repositories = in.readStringArray();
} else {
repositories = new String[]{in.readString()};
}
snapshots = in.readStringArray();
ignoreUnavailable = in.readBoolean();
verbose = in.readBoolean();
Expand All @@ -80,7 +86,11 @@ public GetSnapshotsRequest(StreamInput in) throws IOException {
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeString(repository);
if (out.getVersion().onOrAfter(MULTIPLE_REPOSITORIES_SUPPORT_ADDED)) {
out.writeStringArray(repositories);
} else {
out.writeString(repositories[0]);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This could lead to some strange behaviour in a mixed cluster couldn't it? If I fire a request for multiple snapshots at an older master it'll just respond with the snapshots for a single repo. Also, what about special values like _all here?

Shouldn't we assert repositories.length == 1 here and then add logic to ensure we err out on multiple-repository requests in a mixed version cluster instead of quietly returning "something"?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think, GetSnapshotsResponse.writeTo method protects us from this problem. https://github.com/elastic/elasticsearch/pull/42090/files#diff-4660cd4d86c46cf0b715f6a338e80034R210
So when running 7.latest and 8.x cluster mixed cluster. If 7.latest node receives the request it will generate GetSnapshotResponse and its writeTo method will throw IllegalArgumentException if snapshots from multiple repos are requested.
Because of this special values, like "_all", "*" it's hard to tell in advance if snapshots for multiple repos are requested.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

However, I'm not sure that writeTo/readFrom logic is executed if local transport is used.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If 7.latest node receives the request it will generate GetSnapshotResponse and its writeTo method will throw IllegalArgumentException if snapshots from multiple repos are requested.

Not sure I understand. We are simply writing a truncated request to 7.x nodes here then we won't get failures but simply send a truncated request and get unexpected results (concretely, if we request multiple repos and hit an 8.0 node we will simply get a response containing the snapshots for the first repository and simply ignore the rest won't we?)?

}
out.writeStringArray(snapshots);
out.writeBoolean(ignoreUnavailable);
out.writeBoolean(verbose);
Expand All @@ -89,30 +99,30 @@ public void writeTo(StreamOutput out) throws IOException {
@Override
public ActionRequestValidationException validate() {
ActionRequestValidationException validationException = null;
if (repository == null) {
validationException = addValidationError("repository is missing", validationException);
if (repositories == null || repositories.length == 0) {
validationException = addValidationError("repositories are missing", validationException);
}
return validationException;
}

/**
* Sets repository name
* Sets repository names
*
* @param repository repository name
* @param repositories repository names
* @return this request
*/
public GetSnapshotsRequest repository(String repository) {
this.repository = repository;
public GetSnapshotsRequest repositories(String... repositories) {
this.repositories = repositories;
return this;
}

/**
* Returns repository name
* Returns repository names
*
* @return repository name
* @return repository names
*/
public String repository() {
return this.repository;
public String[] repositories() {
return this.repositories;
}

/**
Expand Down Expand Up @@ -176,4 +186,4 @@ public boolean verbose() {
public void readFrom(StreamInput in) throws IOException {
throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -30,27 +30,20 @@ public class GetSnapshotsRequestBuilder extends MasterNodeOperationRequestBuilde
GetSnapshotsResponse, GetSnapshotsRequestBuilder> {

/**
* Constructs the new get snapshot request
* Constructs the new get snapshot request with specified repositories
*/
public GetSnapshotsRequestBuilder(ElasticsearchClient client, GetSnapshotsAction action) {
super(client, action, new GetSnapshotsRequest());
public GetSnapshotsRequestBuilder(ElasticsearchClient client, GetSnapshotsAction action, String... repositories) {
super(client, action, new GetSnapshotsRequest(repositories));
}

/**
* Constructs the new get snapshot request with specified repository
*/
public GetSnapshotsRequestBuilder(ElasticsearchClient client, GetSnapshotsAction action, String repository) {
super(client, action, new GetSnapshotsRequest(repository));
}

/**
* Sets the repository name
* Sets the repository names
*
* @param repository repository name
* @param repositories repository names
* @return this builder
*/
public GetSnapshotsRequestBuilder setRepository(String repository) {
request.repository(repository);
public GetSnapshotsRequestBuilder setRepositories(String... repositories) {
request.repositories(repositories);
return this;
}

Expand Down
Loading