-
Notifications
You must be signed in to change notification settings - Fork 24.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Support point in time cross cluster search (#61827)
- Loading branch information
Showing
11 changed files
with
371 additions
and
78 deletions.
There are no files selected for viewing
151 changes: 94 additions & 57 deletions
151
server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
106 changes: 106 additions & 0 deletions
106
...re/src/internalClusterTest/java/org/elasticsearch/xpack/core/search/CCSPointInTimeIT.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,106 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License; | ||
| * you may not use this file except in compliance with the Elastic License. | ||
| */ | ||
|
|
||
| package org.elasticsearch.xpack.core.search; | ||
|
|
||
| import org.elasticsearch.action.search.SearchResponse; | ||
| import org.elasticsearch.client.Client; | ||
| import org.elasticsearch.common.unit.TimeValue; | ||
| import org.elasticsearch.index.query.MatchAllQueryBuilder; | ||
| import org.elasticsearch.plugins.Plugin; | ||
| import org.elasticsearch.test.AbstractMultiClustersTestCase; | ||
| import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; | ||
| import org.elasticsearch.xpack.core.search.action.ClosePointInTimeAction; | ||
| import org.elasticsearch.xpack.core.search.action.ClosePointInTimeRequest; | ||
| import org.elasticsearch.xpack.core.search.action.OpenPointInTimeAction; | ||
| import org.elasticsearch.xpack.core.search.action.OpenPointInTimeRequest; | ||
| import org.elasticsearch.xpack.core.search.action.OpenPointInTimeResponse; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.Collection; | ||
| import java.util.List; | ||
|
|
||
| import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; | ||
| import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; | ||
| import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; | ||
|
|
||
| public class CCSPointInTimeIT extends AbstractMultiClustersTestCase { | ||
|
|
||
| @Override | ||
| protected Collection<String> remoteClusterAlias() { | ||
| return List.of("remote_cluster"); | ||
| } | ||
|
|
||
| @Override | ||
| protected Collection<Class<? extends Plugin>> nodePlugins(String clusterAlias) { | ||
| final List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins(clusterAlias)); | ||
| plugins.add(LocalStateCompositeXPackPlugin.class); | ||
| return plugins; | ||
| } | ||
|
|
||
| void indexDocs(Client client, String index, int numDocs) { | ||
| for (int i = 0; i < numDocs; i++) { | ||
| String id = Integer.toString(i); | ||
| client.prepareIndex(index).setId(id).setSource("value", i).get(); | ||
| } | ||
| client.admin().indices().prepareRefresh(index).get(); | ||
| } | ||
|
|
||
| public void testBasic() { | ||
| final Client localClient = client(LOCAL_CLUSTER); | ||
| final Client remoteClient = client("remote_cluster"); | ||
| int localNumDocs = randomIntBetween(10, 50); | ||
| assertAcked(localClient.admin().indices().prepareCreate("local_test")); | ||
| indexDocs(localClient, "local_test", localNumDocs); | ||
|
|
||
| int remoteNumDocs = randomIntBetween(10, 50); | ||
| assertAcked(remoteClient.admin().indices().prepareCreate("remote_test")); | ||
| indexDocs(remoteClient, "remote_test", remoteNumDocs); | ||
| boolean includeLocalIndex = randomBoolean(); | ||
| List<String> indices = new ArrayList<>(); | ||
| if (includeLocalIndex) { | ||
| indices.add( randomFrom("*", "local_*", "local_test")); | ||
| } | ||
| indices.add(randomFrom("*:*", "remote_cluster:*", "remote_cluster:remote_test")); | ||
| String pitId = openPointInTime(indices.toArray(new String[0]), TimeValue.timeValueMinutes(2)); | ||
| try { | ||
| if (randomBoolean()) { | ||
| localClient.prepareIndex("local_test").setId("local_new").setSource().get(); | ||
| localClient.admin().indices().prepareRefresh().get(); | ||
| } | ||
| if (randomBoolean()) { | ||
| remoteClient.prepareIndex("remote_test").setId("remote_new").setSource().get(); | ||
| remoteClient.admin().indices().prepareRefresh().get(); | ||
| } | ||
| SearchResponse resp = localClient.prepareSearch() | ||
| .setPreference(null) | ||
| .setQuery(new MatchAllQueryBuilder()) | ||
| .setSearchContext(pitId, TimeValue.timeValueMinutes(2)) | ||
| .setSize(1000) | ||
| .get(); | ||
| assertNoFailures(resp); | ||
| assertHitCount(resp, (includeLocalIndex ? localNumDocs : 0) + remoteNumDocs); | ||
| } finally { | ||
| closePointInTime(pitId); | ||
| } | ||
| } | ||
|
|
||
| private String openPointInTime(String[] indices, TimeValue keepAlive) { | ||
| OpenPointInTimeRequest request = new OpenPointInTimeRequest( | ||
| indices, | ||
| OpenPointInTimeRequest.DEFAULT_INDICES_OPTIONS, | ||
| keepAlive, | ||
| null, | ||
| null | ||
| ); | ||
| final OpenPointInTimeResponse response = client().execute(OpenPointInTimeAction.INSTANCE, request).actionGet(); | ||
| return response.getSearchContextId(); | ||
| } | ||
|
|
||
| private void closePointInTime(String readerId) { | ||
| client().execute(ClosePointInTimeAction.INSTANCE, new ClosePointInTimeRequest(readerId)).actionGet(); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
113 changes: 113 additions & 0 deletions
113
...-search-security/src/test/resources/rest-api-spec/test/multi_cluster/80_point_in_time.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,113 @@ | ||
| --- | ||
| setup: | ||
| - skip: | ||
| features: headers | ||
|
|
||
| - do: | ||
| cluster.health: | ||
| wait_for_status: yellow | ||
| - do: | ||
| security.put_user: | ||
| username: "joe" | ||
| body: > | ||
| { | ||
| "password": "s3krit", | ||
| "roles" : [ "x_cluster_role" ] | ||
| } | ||
| - do: | ||
| security.put_role: | ||
| name: "x_cluster_role" | ||
| body: > | ||
| { | ||
| "cluster": [], | ||
| "indices": [ | ||
| { | ||
| "names": ["local_pit", "my_remote_cluster:point_in_time_index"], | ||
| "privileges": ["read"] | ||
| } | ||
| ] | ||
| } | ||
| - do: | ||
| security.put_user: | ||
| username: "remote" | ||
| body: > | ||
| { | ||
| "password": "s3krit", | ||
| "roles" : [ "remote_ccs" ] | ||
| } | ||
| - do: | ||
| security.put_role: | ||
| name: "remote_ccs" | ||
| body: > | ||
| { | ||
| } | ||
| --- | ||
| teardown: | ||
| - do: | ||
| security.delete_user: | ||
| username: "joe" | ||
| ignore: 404 | ||
| - do: | ||
| security.delete_role: | ||
| name: "x_cluster_role" | ||
| ignore: 404 | ||
| --- | ||
| "Search with point in time": | ||
|
|
||
| - do: | ||
| indices.create: | ||
| index: local_pit | ||
| body: | ||
| settings: | ||
| index: | ||
| number_of_shards: 2 | ||
| number_of_replicas: 0 | ||
| mappings: | ||
| properties: | ||
| created_at: | ||
| type: date | ||
| format: "yyyy-MM-dd" | ||
| - do: | ||
| bulk: | ||
| refresh: true | ||
| body: | ||
| - '{"index": {"_index": "local_pit"}}' | ||
| - '{"f": "l1", "created_at" : "2020-01-01"}' | ||
| - '{"index": {"_index": "local_pit"}}' | ||
| - '{"f": "l2", "created_at" : "2021-01-02"}' | ||
|
|
||
| - do: | ||
| headers: { Authorization: "Basic am9lOnMza3JpdA==" } | ||
| open_point_in_time: | ||
| index: my_remote_cluster:point_in_time_index,local_pit | ||
| keep_alive: 5m | ||
| - set: {id: pit_id} | ||
|
|
||
| - do: | ||
| headers: { Authorization: "Basic am9lOnMza3JpdA==" } | ||
| search: | ||
| rest_total_hits_as_int: true | ||
| sort: created_at | ||
| body: | ||
| query: | ||
| range: | ||
| created_at: | ||
| gte: "2020-01-03" | ||
| pit: | ||
| id: "$pit_id" | ||
| keep_alive: 1m | ||
|
|
||
| - match: { hits.total: 3 } | ||
| - match: { hits.hits.0._index: "my_remote_cluster:point_in_time_index" } | ||
| - match: { hits.hits.0._source.f: "r3" } | ||
| - match: { hits.hits.1._index: "my_remote_cluster:point_in_time_index" } | ||
| - match: { hits.hits.1._source.f: "r4" } | ||
| - match: { hits.hits.2._index: "local_pit" } | ||
| - match: { hits.hits.2._source.f: "l2" } | ||
|
|
||
| - do: | ||
| headers: { Authorization: "Basic am9lOnMza3JpdA==" } | ||
| close_point_in_time: | ||
| body: | ||
| id: "$pit_id" |
Oops, something went wrong.