Text fields are stored by default in TSDB indices (elastic#106338)

* Text fields are stored by default with synthetic source

Synthetic source requires text fields to be stored or have keyword
sub-field that supports synthetic source. If there are no keyword fields
 users currently have to explicitly set 'store' to 'true' or get a
validation exception. This is not the best experience. It is quite
likely that setting `store` to `true` is  the correct thing to do but
users still get an error and need to investigate it. With this change if
 `store` setting is not specified in such context it  will be set to
 `true` by default. Setting it explicitly to `false` results in the
 exception.

Closes elastic#97039

docs/changelog/106338.yaml

@@ -0,0 +1,6 @@
+pr: 106338
+summary: Text fields are stored by default in TSDB indices
+area: TSDB
+type: enhancement
+issues:
+ - 97039

docs/reference/mapping/types/text.asciidoc

@@ -133,8 +133,11 @@ The following parameters are accepted by `text` fields:
 <<mapping-store,`store`>>::
 
     Whether the field value should be stored and retrievable separately from
-    the <<mapping-source-field,`_source`>> field. Accepts `true` or `false`
-    (default).
+    the <<mapping-source-field,`_source`>> field. Accepts `true` or `false` (default).
+    This parameter will be automatically set to `true` for TSDB indices
+    (indices that have `index.mode` set to `time_series`)
+    if there is no <<keyword-synthetic-source, `keyword`>>
+    sub-field that supports synthetic `_source`.
 
 <<search-analyzer,`search_analyzer`>>::
 

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml

@@ -278,6 +278,7 @@ synthetic source text field:
                                 type: keyword
                           name:
                             type: text
+                            store: false
                       value:
                         type: long
                         time_series_metric: gauge

server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java

@@ -333,7 +333,11 @@ public boolean newDynamicStringField(DocumentParserContext context, String name)
                 );
             } else {
                 return createDynamicField(
-                    new TextFieldMapper.Builder(name, context.indexAnalyzers()).addMultiField(
+                    new TextFieldMapper.Builder(
+                        name,
+                        context.indexAnalyzers(),
+                        context.indexSettings().getMode().isSyntheticSourceEnabled()
+                    ).addMultiField(
                         new KeywordFieldMapper.Builder("keyword", context.indexSettings().getIndexVersionCreated()).ignoreAbove(256)
                     ),
                     context

server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -450,13 +450,28 @@ public static class Builder {
 
             private final Map<String, Function<MapperBuilderContext, FieldMapper>> mapperBuilders = new HashMap<>();
 
+            private boolean hasSyntheticSourceCompatibleKeywordField;
+
             public Builder add(FieldMapper.Builder builder) {
                 mapperBuilders.put(builder.name(), builder::build);
+
+                if (builder instanceof KeywordFieldMapper.Builder kwd) {
+                    if (kwd.hasNormalizer() == false && (kwd.hasDocValues() || kwd.isStored())) {
+                        hasSyntheticSourceCompatibleKeywordField = true;
+                    }
+                }
+
                 return this;
             }
 
             private void add(FieldMapper mapper) {
                 mapperBuilders.put(mapper.simpleName(), context -> mapper);
+
+                if (mapper instanceof KeywordFieldMapper kwd) {
+                    if (kwd.hasNormalizer() == false && (kwd.fieldType().hasDocValues() || kwd.fieldType().isStored())) {
+                        hasSyntheticSourceCompatibleKeywordField = true;
+                    }
+                }
             }
 
             private void update(FieldMapper toMerge, MapperMergeContext context) {
@@ -474,6 +489,10 @@ public boolean hasMultiFields() {
                 return mapperBuilders.isEmpty() == false;
             }
 
+            public boolean hasSyntheticSourceCompatibleKeywordField() {
+                return hasSyntheticSourceCompatibleKeywordField;
+            }
+
             public MultiFields build(Mapper.Builder mainFieldBuilder, MapperBuilderContext context) {
                 if (mapperBuilders.isEmpty()) {
                     return empty();
@@ -1134,6 +1153,10 @@ public static Parameter<Boolean> storeParam(Function<FieldMapper, Boolean> initi
             return Parameter.boolParam("store", false, initializer, defaultValue);
         }
 
+        public static Parameter<Boolean> storeParam(Function<FieldMapper, Boolean> initializer, Supplier<Boolean> defaultValue) {
+            return Parameter.boolParam("store", false, initializer, defaultValue);
+        }
+
         public static Parameter<Boolean> docValuesParam(Function<FieldMapper, Boolean> initializer, boolean defaultValue) {
             return Parameter.boolParam("doc_values", false, initializer, defaultValue);
         }

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -227,6 +227,10 @@ Builder normalizer(String normalizerName) {
             return this;
         }
 
+        public boolean hasNormalizer() {
+            return this.normalizer.get() != null;
+        }
+
         Builder nullValue(String nullValue) {
             this.nullValue.setValue(nullValue);
             return this;
@@ -237,6 +241,10 @@ public Builder docValues(boolean hasDocValues) {
             return this;
         }
 
+        public boolean hasDocValues() {
+            return this.hasDocValues.get();
+        }
+
         public Builder dimension(boolean dimension) {
             this.dimension.setValue(dimension);
             return this;
@@ -247,6 +255,15 @@ public Builder indexed(boolean indexed) {
             return this;
         }
 
+        public Builder stored(boolean stored) {
+            this.stored.setValue(stored);
+            return this;
+        }
+
+        public boolean isStored() {
+            return this.stored.get();
+        }
+
         private FieldValues<String> scriptValues() {
             if (script.get() == null) {
                 return null;

server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java

@@ -236,9 +236,11 @@ private static FielddataFrequencyFilter parseFrequencyFilter(String name, Mappin
     public static class Builder extends FieldMapper.Builder {
 
         private final IndexVersion indexCreatedVersion;
+        private final Parameter<Boolean> store;
+
+        private final boolean isSyntheticSourceEnabledViaIndexMode;
 
         private final Parameter<Boolean> index = Parameter.indexParam(m -> ((TextFieldMapper) m).index, true);
-        private final Parameter<Boolean> store = Parameter.storeParam(m -> ((TextFieldMapper) m).store, false);
 
         final Parameter<SimilarityProvider> similarity = TextParams.similarity(m -> ((TextFieldMapper) m).similarity);
 
@@ -283,19 +285,36 @@ public static class Builder extends FieldMapper.Builder {
 
         final TextParams.Analyzers analyzers;
 
-        public Builder(String name, IndexAnalyzers indexAnalyzers) {
-            this(name, IndexVersion.current(), indexAnalyzers);
+        public Builder(String name, IndexAnalyzers indexAnalyzers, boolean isSyntheticSourceEnabledViaIndexMode) {
+            this(name, IndexVersion.current(), indexAnalyzers, isSyntheticSourceEnabledViaIndexMode);
         }
 
-        public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers indexAnalyzers) {
+        public Builder(
+            String name,
+            IndexVersion indexCreatedVersion,
+            IndexAnalyzers indexAnalyzers,
+            boolean isSyntheticSourceEnabledViaIndexMode
+        ) {
             super(name);
+
+            // If synthetic source is used we need to either store this field
+            // to recreate the source or use keyword multi-fields for that.
+            // So if there are no suitable multi-fields we will default to
+            // storing the field without requiring users to explicitly set 'store'.
+            //
+            // If 'store' parameter was explicitly provided we'll reject the request.
+            this.store = Parameter.storeParam(
+                m -> ((TextFieldMapper) m).store,
+                () -> isSyntheticSourceEnabledViaIndexMode && multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false
+            );
             this.indexCreatedVersion = indexCreatedVersion;
             this.analyzers = new TextParams.Analyzers(
                 indexAnalyzers,
                 m -> ((TextFieldMapper) m).indexAnalyzer,
                 m -> (((TextFieldMapper) m).positionIncrementGap),
                 indexCreatedVersion
             );
+            this.isSyntheticSourceEnabledViaIndexMode = isSyntheticSourceEnabledViaIndexMode;
         }
 
         public Builder index(boolean index) {
@@ -387,13 +406,9 @@ private static KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate(Field
             if (fieldType.stored()) {
                 return null;
             }
-            for (Mapper sub : multiFields) {
-                if (sub.typeName().equals(KeywordFieldMapper.CONTENT_TYPE)) {
-                    KeywordFieldMapper kwd = (KeywordFieldMapper) sub;
-                    if (kwd.hasNormalizer() == false && (kwd.fieldType().hasDocValues() || kwd.fieldType().isStored())) {
-                        return kwd.fieldType();
-                    }
-                }
+            var kwd = getKeywordFieldMapperForSyntheticSource(multiFields);
+            if (kwd != null) {
+                return kwd.fieldType();
             }
             return null;
         }
@@ -483,7 +498,7 @@ public TextFieldMapper build(MapperBuilderContext context) {
     private static final IndexVersion MINIMUM_COMPATIBILITY_VERSION = IndexVersion.fromId(5000099);
 
     public static final TypeParser PARSER = new TypeParser(
-        (n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers()),
+        (n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), c.getIndexSettings().getMode().isSyntheticSourceEnabled()),
         MINIMUM_COMPATIBILITY_VERSION
     );
 
@@ -1203,6 +1218,8 @@ public Query existsQuery(SearchExecutionContext context) {
     private final SubFieldInfo prefixFieldInfo;
     private final SubFieldInfo phraseFieldInfo;
 
+    private final boolean isSyntheticSourceEnabledViaIndexMode;
+
     private TextFieldMapper(
         String simpleName,
         FieldType fieldType,
@@ -1235,6 +1252,7 @@ private TextFieldMapper(
         this.indexPrefixes = builder.indexPrefixes.getValue();
         this.freqFilter = builder.freqFilter.getValue();
         this.fieldData = builder.fieldData.get();
+        this.isSyntheticSourceEnabledViaIndexMode = builder.isSyntheticSourceEnabledViaIndexMode;
     }
 
     @Override
@@ -1258,7 +1276,7 @@ public Map<String, NamedAnalyzer> indexAnalyzers() {
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(simpleName(), indexCreatedVersion, indexAnalyzers).init(this);
+        return new Builder(simpleName(), indexCreatedVersion, indexAnalyzers, isSyntheticSourceEnabledViaIndexMode).init(this);
     }
 
     @Override
@@ -1454,15 +1472,12 @@ protected void write(XContentBuilder b, Object value) throws IOException {
                 }
             };
         }
-        for (Mapper sub : this) {
-            if (sub.typeName().equals(KeywordFieldMapper.CONTENT_TYPE)) {
-                KeywordFieldMapper kwd = (KeywordFieldMapper) sub;
-                if (kwd.hasNormalizer() == false && (kwd.fieldType().hasDocValues() || kwd.fieldType().isStored())) {
 
-                    return kwd.syntheticFieldLoader(simpleName());
-                }
-            }
+        var kwd = getKeywordFieldMapperForSyntheticSource(this);
+        if (kwd != null) {
+            return kwd.syntheticFieldLoader(simpleName());
         }
+
         throw new IllegalArgumentException(
             String.format(
                 Locale.ROOT,
@@ -1473,4 +1488,17 @@ protected void write(XContentBuilder b, Object value) throws IOException {
             )
         );
     }
+
+    private static KeywordFieldMapper getKeywordFieldMapperForSyntheticSource(Iterable<? extends Mapper> multiFields) {
+        for (Mapper sub : multiFields) {
+            if (sub.typeName().equals(KeywordFieldMapper.CONTENT_TYPE)) {
+                KeywordFieldMapper kwd = (KeywordFieldMapper) sub;
+                if (kwd.hasNormalizer() == false && (kwd.fieldType().hasDocValues() || kwd.fieldType().isStored())) {
+                    return kwd;
+                }
+            }
+        }
+
+        return null;
+    }
 }

server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java

@@ -196,7 +196,11 @@ MappedFieldType failIfFieldMappingNotFound(String name, MappedFieldType fieldMap
         if (fieldMapping != null || allowUnmappedFields) {
             return fieldMapping;
         } else if (mapUnmappedFieldAsString) {
-            TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name, getIndexAnalyzers());
+            TextFieldMapper.Builder builder = new TextFieldMapper.Builder(
+                name,
+                getIndexAnalyzers(),
+                getIndexSettings() != null && getIndexSettings().getMode().isSyntheticSourceEnabled()
+            );
             return builder.build(MapperBuilderContext.root(false, false)).fieldType();
         } else {
             throw new QueryShardException(this, "No field mapping can be found for the field with name [{}]", name);

server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java

@@ -90,9 +90,11 @@ public <IFD extends IndexFieldData<?>> IFD getForField(String type, String field
             if (docValues) {
                 fieldType = new KeywordFieldMapper.Builder(fieldName, IndexVersion.current()).build(context).fieldType();
             } else {
-                fieldType = new TextFieldMapper.Builder(fieldName, createDefaultIndexAnalyzers()).fielddata(true)
-                    .build(context)
-                    .fieldType();
+                fieldType = new TextFieldMapper.Builder(
+                    fieldName,
+                    createDefaultIndexAnalyzers(),
+                    indexService.getIndexSettings().getMode().isSyntheticSourceEnabled()
+                ).fielddata(true).build(context).fieldType();
             }
         } else if (type.equals("float")) {
             fieldType = new NumberFieldMapper.Builder(

server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java

@@ -52,10 +52,11 @@ public void testFilterByFrequency() throws Exception {
 
         {
             indexService.clearCaches(false, true);
-            MappedFieldType ft = new TextFieldMapper.Builder("high_freq", createDefaultIndexAnalyzers()).fielddata(true)
-                .fielddataFrequencyFilter(0, random.nextBoolean() ? 100 : 0.5d, 0)
-                .build(builderContext)
-                .fieldType();
+            MappedFieldType ft = new TextFieldMapper.Builder(
+                "high_freq",
+                createDefaultIndexAnalyzers(),
+                indexService.getIndexSettings().getMode().isSyntheticSourceEnabled()
+            ).fielddata(true).fielddataFrequencyFilter(0, random.nextBoolean() ? 100 : 0.5d, 0).build(builderContext).fieldType();
             IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft, MappedFieldType.FielddataOperation.SEARCH);
             for (LeafReaderContext context : contexts) {
                 LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context);
@@ -67,7 +68,11 @@ public void testFilterByFrequency() throws Exception {
         }
         {
             indexService.clearCaches(false, true);
-            MappedFieldType ft = new TextFieldMapper.Builder("high_freq", createDefaultIndexAnalyzers()).fielddata(true)
+            MappedFieldType ft = new TextFieldMapper.Builder(
+                "high_freq",
+                createDefaultIndexAnalyzers(),
+                indexService.getIndexSettings().getMode().isSyntheticSourceEnabled()
+            ).fielddata(true)
                 .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, 201, 100)
                 .build(builderContext)
                 .fieldType();
@@ -82,7 +87,11 @@ public void testFilterByFrequency() throws Exception {
 
         {
             indexService.clearCaches(false, true);// test # docs with value
-            MappedFieldType ft = new TextFieldMapper.Builder("med_freq", createDefaultIndexAnalyzers()).fielddata(true)
+            MappedFieldType ft = new TextFieldMapper.Builder(
+                "med_freq",
+                createDefaultIndexAnalyzers(),
+                indexService.getIndexSettings().getMode().isSyntheticSourceEnabled()
+            ).fielddata(true)
                 .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, Integer.MAX_VALUE, 101)
                 .build(builderContext)
                 .fieldType();
@@ -98,7 +107,11 @@ public void testFilterByFrequency() throws Exception {
 
         {
             indexService.clearCaches(false, true);
-            MappedFieldType ft = new TextFieldMapper.Builder("med_freq", createDefaultIndexAnalyzers()).fielddata(true)
+            MappedFieldType ft = new TextFieldMapper.Builder(
+                "med_freq",
+                createDefaultIndexAnalyzers(),
+                indexService.getIndexSettings().getMode().isSyntheticSourceEnabled()
+            ).fielddata(true)
                 .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, Integer.MAX_VALUE, 101)
                 .build(builderContext)
                 .fieldType();

server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java

@@ -156,12 +156,16 @@ public void testClearField() throws Exception {
         );
 
         final MapperBuilderContext context = MapperBuilderContext.root(false, false);
-        final MappedFieldType mapper1 = new TextFieldMapper.Builder("field_1", createDefaultIndexAnalyzers()).fielddata(true)
-            .build(context)
-            .fieldType();
-        final MappedFieldType mapper2 = new TextFieldMapper.Builder("field_2", createDefaultIndexAnalyzers()).fielddata(true)
-            .build(context)
-            .fieldType();
+        final MappedFieldType mapper1 = new TextFieldMapper.Builder(
+            "field_1",
+            createDefaultIndexAnalyzers(),
+            indexService.getIndexSettings().getMode().isSyntheticSourceEnabled()
+        ).fielddata(true).build(context).fieldType();
+        final MappedFieldType mapper2 = new TextFieldMapper.Builder(
+            "field_2",
+            createDefaultIndexAnalyzers(),
+            indexService.getIndexSettings().getMode().isSyntheticSourceEnabled()
+        ).fielddata(true).build(context).fieldType();
         final IndexWriter writer = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new KeywordAnalyzer()));
         Document doc = new Document();
         doc.add(new StringField("field_1", "thisisastring", Store.NO));
@@ -223,9 +227,11 @@ public void testFieldDataCacheListener() throws Exception {
         );
 
         final MapperBuilderContext context = MapperBuilderContext.root(false, false);
-        final MappedFieldType mapper1 = new TextFieldMapper.Builder("s", createDefaultIndexAnalyzers()).fielddata(true)
-            .build(context)
-            .fieldType();
+        final MappedFieldType mapper1 = new TextFieldMapper.Builder(
+            "s",
+            createDefaultIndexAnalyzers(),
+            indexService.getIndexSettings().getMode().isSyntheticSourceEnabled()
+        ).fielddata(true).build(context).fieldType();
         final IndexWriter writer = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new KeywordAnalyzer()));
         Document doc = new Document();
         doc.add(new StringField("s", "thisisastring", Store.NO));