From 675c1f43cf46286ba59c04b4b82f78595d02a685 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Wed, 20 Oct 2021 00:05:54 +0200 Subject: [PATCH] Store Template's mappings as bytes for disk serialization (#78746) This change the way we store mappings in `Template` during serialization to disk - instead storing it as map we use byte array that we already have. This avoids deserialization-serialization cycle during storing cluster state on disk. --- .../GetComponentTemplatesResponseTests.java | 3 +- ...ComposableIndexTemplatesResponseTests.java | 3 +- .../ElasticsearchNodeCommand.java | 14 +++++--- .../cluster/metadata/ComponentTemplate.java | 2 +- .../metadata/ComponentTemplateMetadata.java | 2 +- .../metadata/ComposableIndexTemplate.java | 4 +-- .../ComposableIndexTemplateMetadata.java | 2 +- .../cluster/metadata/Template.java | 32 +++++++++++++++---- 8 files changed, 43 insertions(+), 19 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComponentTemplatesResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComponentTemplatesResponseTests.java index 8099370e02fa9..5f5070d3f4a11 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComponentTemplatesResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComponentTemplatesResponseTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.test.ESTestCase; @@ -79,7 +80,7 @@ private static void toXContent(GetComponentTemplatesResponse response, XContentB builder.startObject(); builder.field("name", e.getKey()); builder.field("component_template"); - e.getValue().toXContent(builder, null); + e.getValue().toXContent(builder, ToXContent.EMPTY_PARAMS); builder.endObject(); } builder.endArray(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java index b93af425dd932..a241fb235b2b6 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -53,7 +54,7 @@ private static void toXContent(GetComposableIndexTemplatesResponse response, XCo builder.startObject(); builder.field("name", e.getKey()); builder.field("index_template"); - e.getValue().toXContent(builder, null); + e.getValue().toXContent(builder, ToXContent.EMPTY_PARAMS); builder.endObject(); } builder.endArray(); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java index f4041e76218ca..21a965f3fc60f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java @@ -9,6 +9,7 @@ import joptsimple.OptionParser; import joptsimple.OptionSet; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.store.LockObtainFailedException; @@ -22,20 +23,22 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.Diff; +import org.elasticsearch.cluster.metadata.ComponentTemplateMetadata; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplateMetadata; import org.elasticsearch.cluster.metadata.DataStreamMetadata; import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.core.Tuple; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.core.Tuple; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.NodeMetadata; import org.elasticsearch.gateway.PersistedClusterStateService; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; import java.nio.file.Files; @@ -68,7 +71,8 @@ public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand { public T parseNamedObject(Class categoryClass, String name, XContentParser parser, C context) throws IOException { // Currently, two unknown top-level objects are present if (Metadata.Custom.class.isAssignableFrom(categoryClass)) { - if (DataStreamMetadata.TYPE.equals(name)) { + if (DataStreamMetadata.TYPE.equals(name) || ComposableIndexTemplateMetadata.TYPE.equals(name) + || ComponentTemplateMetadata.TYPE.equals(name)) { // DataStreamMetadata is used inside Metadata class for validation purposes and building the indicesLookup, // therefor even es node commands need to be able to parse it. return super.parseNamedObject(categoryClass, name, parser, context); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java index 2548431276020..ecfece366f7ad 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java @@ -129,7 +129,7 @@ public String toString() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(TEMPLATE.getPreferredName(), this.template); + builder.field(TEMPLATE.getPreferredName(), this.template, params); if (this.version != null) { builder.field(VERSION.getPreferredName(), this.version); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java index c655145bd7ff0..883372b9a4638 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java @@ -98,7 +98,7 @@ public static ComponentTemplateMetadata fromXContent(XContentParser parser) thro public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(COMPONENT_TEMPLATE.getPreferredName()); for (Map.Entry template : componentTemplates.entrySet()) { - builder.field(template.getKey(), template.getValue()); + builder.field(template.getKey(), template.getValue(), params); } builder.endObject(); return builder; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index 80ab79c2b24ad..79581380d296b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -202,7 +202,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.stringListField(INDEX_PATTERNS.getPreferredName(), this.indexPatterns); if (this.template != null) { - builder.field(TEMPLATE.getPreferredName(), this.template); + builder.field(TEMPLATE.getPreferredName(), this.template, params); } if (this.componentTemplates != null) { builder.stringListField(COMPOSED_OF.getPreferredName(), this.componentTemplates); @@ -217,7 +217,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(METADATA.getPreferredName(), metadata); } if (this.dataStreamTemplate != null) { - builder.field(DATA_STREAM.getPreferredName(), dataStreamTemplate); + builder.field(DATA_STREAM.getPreferredName(), dataStreamTemplate, params); } if (this.allowAutoCreate != null) { builder.field(ALLOW_AUTO_CREATE.getPreferredName(), allowAutoCreate); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java index 67fd84779f6b7..1fe3f3ab8d8c2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java @@ -99,7 +99,7 @@ public void writeTo(StreamOutput out) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(INDEX_TEMPLATE.getPreferredName()); for (Map.Entry template : indexTemplates.entrySet()) { - builder.field(template.getKey(), template.getValue()); + builder.field(template.getKey(), template.getValue(), params); } builder.endObject(); return builder; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java index ffb23b6835b7b..8e7833aae6ee1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java @@ -11,6 +11,7 @@ import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.Nullable; +import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; @@ -27,6 +28,7 @@ import org.elasticsearch.index.mapper.MapperService; import java.io.IOException; +import java.util.Base64; import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -47,8 +49,18 @@ public class Template extends AbstractDiffable