From 2029f4731ce6255945a6d25cfc01cc9dc6da2c64 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 7 Oct 2021 07:25:50 +0100 Subject: [PATCH 1/4] Adjust /_cat/templates not to request all metadata Today `GET /_cat/templates` retrieves the whole cluster metadata from the master, which includes all sorts of unnecessary junk and consumes significant resources. This commit reimplements these endpoints using `GetIndexTemplatesAction` and `GetComposableIndexTemplateAction` which are much more efficient. The docs for this API indicate that it accepts a comma-separated list of template names/patterns of the form `GET /_cat/templates/name1,name2` but in fact today it only accepts a single name or pattern. This commit also adds support for multiple names/patterns as the docs claim. --- .../test/cat.templates/20_matching.yml | 179 ++++++++++++++++++ .../rest/action/cat/RestTemplatesAction.java | 138 ++++++++++---- 2 files changed, 282 insertions(+), 35 deletions(-) create mode 100644 rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.templates/20_matching.yml diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.templates/20_matching.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.templates/20_matching.yml new file mode 100644 index 0000000000000..8c3b11d1c8cb3 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.templates/20_matching.yml @@ -0,0 +1,179 @@ +setup: + - do: + indices.put_template: + name: legacy-1 + body: + order: 12 + version: 3 + index_patterns: foo* + + - do: + indices.put_template: + name: legacy-2 + body: + order: 45 + version: 6 + index_patterns: + - bar* + - baz* + + - do: + cluster.put_component_template: + name: test-component-template + body: + template: + settings: + number_of_shards: 1 + number_of_replicas: 0 + + - do: + indices.put_index_template: + name: composable-1 + body: + index_patterns: + - quux* + priority: 78 + version: 9 + composed_of: + - test-component-template + + - do: + indices.put_index_template: + name: composable-2 + body: + index_patterns: + - gruly* + priority: 99 + version: 1 + composed_of: + - test-component-template + +--- +"Matching all templates": + + - do: + cat.templates: + h: [name] + s: [name] + + - match: + $body: /^composable-1\ncomposable-2\nlegacy-1\nlegacy-2\n$/ + + - do: + cat.templates: + name: "x,*" + h: [name] + s: [name] + + - match: + $body: /^composable-1\ncomposable-2\nlegacy-1\nlegacy-2\n$/ + +--- +"Matching all templates with other patterns": + - skip: + version: " - 7.99.99" + reason: "support for multiple patterns added in 8.0.0" + + - do: + cat.templates: + name: "nonexistent*,*,other-name" + h: [name] + s: [name] + + - match: + $body: /^composable-1\ncomposable-2\nlegacy-1\nlegacy-2\n$/ + +--- +"Matching no templates": + + - do: + cat.templates: + name: "nonexistent" + h: [name] + s: [name] + + - match: + $body: /^$/ + +--- +"Matching single names": + + - do: + cat.templates: + name: "legacy-1" + h: [name] + s: [name] + + - match: + $body: /^legacy-1\n$/ + + + - do: + cat.templates: + name: "composable-2" + h: [name] + s: [name] + + - match: + $body: /^composable-2\n$/ + +--- +"Matching single patterns": + + - do: + cat.templates: + name: "legacy-*" + h: [name] + s: [name] + + - match: + $body: /^legacy-1\nlegacy-2\n$/ + + + - do: + cat.templates: + name: "*-2" + h: [name] + s: [name] + + - match: + $body: /^composable-2\nlegacy-2\n$/ + +--- +"Matching lists of names": + - skip: + version: " - 7.99.99" + reason: "support for multiple patterns added in 8.0.0" + + - do: + cat.templates: + name: "legacy-1,composable-2" + h: [name] + s: [name] + + - match: + $body: /^composable-2\nlegacy-1\n$/ + +--- +"Matching names and wildcards": + - skip: + version: " - 7.99.99" + reason: "support for multiple patterns added in 8.0.0" + + - do: + cat.templates: + name: "legacy-1,composable-*" + h: [name] + s: [name] + + - match: + $body: /^composable-1\ncomposable-2\nlegacy-1\n$/ + + - do: + cat.templates: + name: "legacy-*,composable-2" + h: [name] + s: [name] + + - match: + $body: /^composable-2\nlegacy-1\nlegacy-2\n$/ diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestTemplatesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestTemplatesAction.java index c4785df10d310..a4496dae997b2 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestTemplatesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestTemplatesAction.java @@ -8,21 +8,27 @@ package org.elasticsearch.rest.action.cat; -import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; -import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.StepListener; +import org.elasticsearch.action.admin.indices.template.get.GetComposableIndexTemplateAction; +import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest; +import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; -import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.Table; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.action.RestResponseListener; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Predicate; import static org.elasticsearch.rest.RestRequest.Method.GET; @@ -47,18 +53,41 @@ protected void documentation(StringBuilder sb) { @Override protected RestChannelConsumer doCatRequest(final RestRequest request, NodeClient client) { - final String matchPattern = request.hasParam("name") ? request.param("name") : null; - final ClusterStateRequest clusterStateRequest = new ClusterStateRequest(); - clusterStateRequest.clear().metadata(true); - clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local())); - clusterStateRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterStateRequest.masterNodeTimeout())); - - return channel -> client.admin().cluster().state(clusterStateRequest, new RestResponseListener(channel) { - @Override - public RestResponse buildResponse(ClusterStateResponse clusterStateResponse) throws Exception { - return RestTable.buildResponse(buildTable(request, clusterStateResponse, matchPattern), channel); - } - }); + final String[] templateNames = Strings.splitStringByCommaToArray(request.param("name", "")); + + final GetIndexTemplatesRequest getIndexTemplatesRequest = new GetIndexTemplatesRequest(templateNames); + getIndexTemplatesRequest.local(request.paramAsBoolean("local", getIndexTemplatesRequest.local())); + getIndexTemplatesRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getIndexTemplatesRequest.masterNodeTimeout())); + final StepListener getIndexTemplatesStep = new StepListener<>(); + + final GetComposableIndexTemplateAction.Request getComposableTemplatesRequest + = new GetComposableIndexTemplateAction.Request(); + getComposableTemplatesRequest.local(request.paramAsBoolean("local", getComposableTemplatesRequest.local())); + getComposableTemplatesRequest.masterNodeTimeout( + request.paramAsTime("master_timeout", getComposableTemplatesRequest.masterNodeTimeout())); + final StepListener getComposableTemplatesStep = new StepListener<>(); + + return channel -> { + client.admin().indices().getTemplates(getIndexTemplatesRequest, getIndexTemplatesStep); + client.execute(GetComposableIndexTemplateAction.INSTANCE, getComposableTemplatesRequest, getComposableTemplatesStep); + + final ActionListener tableListener = new RestResponseListener<>(channel) { + @Override + public RestResponse buildResponse(Table table) throws Exception { + return RestTable.buildResponse(table, channel); + } + }; + + getIndexTemplatesStep.whenComplete(getIndexTemplatesResponse -> + getComposableTemplatesStep.whenComplete(getComposableIndexTemplatesResponse -> + ActionListener.completeWith(tableListener, () -> buildTable( + request, + getIndexTemplatesResponse, + getComposableIndexTemplatesResponse, + templateNames) + ), tableListener::onFailure + ), tableListener::onFailure); + }; } @Override @@ -74,26 +103,30 @@ protected Table getTableWithHeader(RestRequest request) { return table; } - private Table buildTable(RestRequest request, ClusterStateResponse clusterStateResponse, String patternString) { - Table table = getTableWithHeader(request); - Metadata metadata = clusterStateResponse.getState().metadata(); - for (ObjectObjectCursor entry : metadata.templates()) { - IndexTemplateMetadata indexData = entry.value; - if (patternString == null || Regex.simpleMatch(patternString, indexData.name())) { - table.startRow(); - table.addCell(indexData.name()); - table.addCell("[" + String.join(", ", indexData.patterns()) + "]"); - table.addCell(indexData.getOrder()); - table.addCell(indexData.getVersion()); - table.addCell(""); - table.endRow(); - } + private Table buildTable( + RestRequest request, + GetIndexTemplatesResponse getIndexTemplatesResponse, + GetComposableIndexTemplateAction.Response getComposableIndexTemplatesResponse, + String[] requestedNames + ) { + final Predicate namePredicate = getNamePredicate(requestedNames); + + final Table table = getTableWithHeader(request); + for (IndexTemplateMetadata indexData : getIndexTemplatesResponse.getIndexTemplates()) { + assert namePredicate.test(indexData.getName()); + table.startRow(); + table.addCell(indexData.name()); + table.addCell("[" + String.join(", ", indexData.patterns()) + "]"); + table.addCell(indexData.getOrder()); + table.addCell(indexData.getVersion()); + table.addCell(""); + table.endRow(); } - for (Map.Entry entry : metadata.templatesV2().entrySet()) { - String name = entry.getKey(); - ComposableIndexTemplate template = entry.getValue(); - if (patternString == null || Regex.simpleMatch(patternString, name)) { + for (Map.Entry entry : getComposableIndexTemplatesResponse.indexTemplates().entrySet()) { + final String name = entry.getKey(); + if (namePredicate.test(name)) { + final ComposableIndexTemplate template = entry.getValue(); table.startRow(); table.addCell(name); table.addCell("[" + String.join(", ", template.indexPatterns()) + "]"); @@ -103,6 +136,41 @@ private Table buildTable(RestRequest request, ClusterStateResponse clusterStateR table.endRow(); } } + return table; } + + private Predicate getNamePredicate(String[] requestedNames) { + if (requestedNames.length == 0) { + return name -> true; + } + + final Set exactMatches = new HashSet<>(); + final List patterns = new ArrayList<>(); + for (String requestedName : requestedNames) { + if (Regex.isMatchAllPattern(requestedName)) { + return name -> true; + } else if (Regex.isSimpleMatchPattern(requestedName)) { + patterns.add(requestedName); + } else { + exactMatches.add(requestedName); + } + } + + if (patterns.isEmpty()) { + return exactMatches::contains; + } + + return name -> { + if (exactMatches.contains(name)) { + return true; + } + for (String pattern : patterns) { + if (Regex.simpleMatch(pattern, name)) { + return true; + } + } + return false; + }; + } } From e182ff2e53e11045deabcfdfc3fc528676897995 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 7 Oct 2021 08:29:04 +0100 Subject: [PATCH 2/4] Deal with extra templates created sometimes --- .../test/cat.templates/20_matching.yml | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.templates/20_matching.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.templates/20_matching.yml index 8c3b11d1c8cb3..c551439a3a8dd 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.templates/20_matching.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.templates/20_matching.yml @@ -1,7 +1,7 @@ setup: - do: indices.put_template: - name: legacy-1 + name: test-legacy-1 body: order: 12 version: 3 @@ -9,7 +9,7 @@ setup: - do: indices.put_template: - name: legacy-2 + name: test-legacy-2 body: order: 45 version: 6 @@ -28,7 +28,7 @@ setup: - do: indices.put_index_template: - name: composable-1 + name: test-composable-1 body: index_patterns: - quux* @@ -39,7 +39,7 @@ setup: - do: indices.put_index_template: - name: composable-2 + name: test-composable-2 body: index_patterns: - gruly* @@ -57,7 +57,7 @@ setup: s: [name] - match: - $body: /^composable-1\ncomposable-2\nlegacy-1\nlegacy-2\n$/ + $body: /test-composable-1\ntest-composable-2\ntest-legacy-1\ntest-legacy-2\n/ - do: cat.templates: @@ -66,7 +66,7 @@ setup: s: [name] - match: - $body: /^composable-1\ncomposable-2\nlegacy-1\nlegacy-2\n$/ + $body: /test-composable-1\ntest-composable-2\ntest-legacy-1\ntest-legacy-2\n/ --- "Matching all templates with other patterns": @@ -81,7 +81,7 @@ setup: s: [name] - match: - $body: /^composable-1\ncomposable-2\nlegacy-1\nlegacy-2\n$/ + $body: /test-composable-1\ntest-composable-2\ntest-legacy-1\ntest-legacy-2\n/ --- "Matching no templates": @@ -100,44 +100,44 @@ setup: - do: cat.templates: - name: "legacy-1" + name: "test-legacy-1" h: [name] s: [name] - match: - $body: /^legacy-1\n$/ + $body: /^test-legacy-1\n$/ - do: cat.templates: - name: "composable-2" + name: "test-composable-2" h: [name] s: [name] - match: - $body: /^composable-2\n$/ + $body: /^test-composable-2\n$/ --- "Matching single patterns": - do: cat.templates: - name: "legacy-*" + name: "test-legacy-*" h: [name] s: [name] - match: - $body: /^legacy-1\nlegacy-2\n$/ + $body: /^test-legacy-1\ntest-legacy-2\n$/ - do: cat.templates: - name: "*-2" + name: "test-*-2" h: [name] s: [name] - match: - $body: /^composable-2\nlegacy-2\n$/ + $body: /^test-composable-2\ntest-legacy-2\n$/ --- "Matching lists of names": @@ -147,12 +147,12 @@ setup: - do: cat.templates: - name: "legacy-1,composable-2" + name: "test-legacy-1,test-composable-2" h: [name] s: [name] - match: - $body: /^composable-2\nlegacy-1\n$/ + $body: /^test-composable-2\ntest-legacy-1\n$/ --- "Matching names and wildcards": @@ -162,18 +162,18 @@ setup: - do: cat.templates: - name: "legacy-1,composable-*" + name: "test-legacy-1,test-composable-*" h: [name] s: [name] - match: - $body: /^composable-1\ncomposable-2\nlegacy-1\n$/ + $body: /^test-composable-1\ntest-composable-2\ntest-legacy-1\n$/ - do: cat.templates: - name: "legacy-*,composable-2" + name: "test-legacy-*,test-composable-2" h: [name] s: [name] - match: - $body: /^composable-2\nlegacy-1\nlegacy-2\n$/ + $body: /^test-composable-2\ntest-legacy-1\ntest-legacy-2\n$/ From af780cdff82582409a8bf0171564d087b34abab3 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 7 Oct 2021 08:56:57 +0100 Subject: [PATCH 3/4] bwc --- .../resources/rest-api-spec/test/cat.templates/20_matching.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.templates/20_matching.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.templates/20_matching.yml index c551439a3a8dd..0ee0cd3fb4af2 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.templates/20_matching.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.templates/20_matching.yml @@ -61,7 +61,7 @@ setup: - do: cat.templates: - name: "x,*" + name: "*" h: [name] s: [name] From d71f295ed694d4c938140396e249e855279dd3dd Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 7 Oct 2021 11:00:29 +0100 Subject: [PATCH 4/4] Construct StepListeners later --- .../elasticsearch/rest/action/cat/RestTemplatesAction.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestTemplatesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestTemplatesAction.java index a4496dae997b2..ba24839e10f9c 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestTemplatesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestTemplatesAction.java @@ -58,17 +58,19 @@ protected RestChannelConsumer doCatRequest(final RestRequest request, NodeClient final GetIndexTemplatesRequest getIndexTemplatesRequest = new GetIndexTemplatesRequest(templateNames); getIndexTemplatesRequest.local(request.paramAsBoolean("local", getIndexTemplatesRequest.local())); getIndexTemplatesRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getIndexTemplatesRequest.masterNodeTimeout())); - final StepListener getIndexTemplatesStep = new StepListener<>(); final GetComposableIndexTemplateAction.Request getComposableTemplatesRequest = new GetComposableIndexTemplateAction.Request(); getComposableTemplatesRequest.local(request.paramAsBoolean("local", getComposableTemplatesRequest.local())); getComposableTemplatesRequest.masterNodeTimeout( request.paramAsTime("master_timeout", getComposableTemplatesRequest.masterNodeTimeout())); - final StepListener getComposableTemplatesStep = new StepListener<>(); return channel -> { + + final StepListener getIndexTemplatesStep = new StepListener<>(); client.admin().indices().getTemplates(getIndexTemplatesRequest, getIndexTemplatesStep); + + final StepListener getComposableTemplatesStep = new StepListener<>(); client.execute(GetComposableIndexTemplateAction.INSTANCE, getComposableTemplatesRequest, getComposableTemplatesStep); final ActionListener
tableListener = new RestResponseListener<>(channel) {