diff --git a/docs/changelog/89251.yaml b/docs/changelog/89251.yaml new file mode 100644 index 0000000000000..a3285d7b467a5 --- /dev/null +++ b/docs/changelog/89251.yaml @@ -0,0 +1,6 @@ +pr: 89251 +summary: Include runtime fields in total fields count +area: Mapping +type: bug +issues: + - 88265 diff --git a/docs/reference/mapping/mapping-settings-limit.asciidoc b/docs/reference/mapping/mapping-settings-limit.asciidoc index 0f94a376f4041..c499ca7675f2c 100644 --- a/docs/reference/mapping/mapping-settings-limit.asciidoc +++ b/docs/reference/mapping/mapping-settings-limit.asciidoc @@ -4,7 +4,8 @@ Use the following settings to limit the number of field mappings (created manual `index.mapping.total_fields.limit`:: The maximum number of fields in an index. Field and object mappings, as well as - field aliases count towards this limit. The default value is `1000`. + field aliases count towards this limit. Mapped runtime fields count towards this + limit as well. The default value is `1000`. + [IMPORTANT] ==== diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java index 4e2b3fb952164..d31475a172056 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java @@ -18,6 +18,7 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Randomness; @@ -213,6 +214,71 @@ public void onFailure(Exception e) { } } + public void testTotalFieldsLimitWithRuntimeFields() { + Settings indexSettings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 4) + .build(); + + String mapping = """ + { + "dynamic":"runtime", + "runtime": { + "my_object.rfield1": { + "type": "keyword" + }, + "rfield2": { + "type": "keyword" + } + }, + "properties": { + "field3" : { + "type": "keyword" + } + } + } + """; + + client().admin().indices().prepareCreate("index1").setSettings(indexSettings).setMapping(mapping).get(); + ensureGreen("index1"); + + { + // introduction of a new object with 2 new sub-fields fails + final IndexRequestBuilder indexRequestBuilder = client().prepareIndex("index1") + .setId("1") + .setSource("field3", "value3", "my_object2", Map.of("new_field1", "value1", "new_field2", "value2")); + Exception exc = expectThrows(MapperParsingException.class, () -> indexRequestBuilder.get(TimeValue.timeValueSeconds(10))); + assertThat(exc.getMessage(), Matchers.containsString("failed to parse")); + assertThat(exc.getCause(), instanceOf(IllegalArgumentException.class)); + assertThat( + exc.getCause().getMessage(), + Matchers.containsString("Limit of total fields [4] has been exceeded while adding new fields [2]") + ); + } + + { + // introduction of a new single field succeeds + client().prepareIndex("index1").setId("2").setSource("field3", "value3", "new_field4", 100).get(); + } + + { + // remove 2 runtime field mappings + assertAcked(client().admin().indices().preparePutMapping("index1").setSource(""" + { + "runtime": { + "my_object.rfield1": null, + "rfield2" : null + } + } + """, XContentType.JSON)); + + // introduction of a new object with 2 new sub-fields succeeds + client().prepareIndex("index1") + .setId("1") + .setSource("field3", "value3", "my_object2", Map.of("new_field1", "value1", "new_field2", "value2")); + } + } + public void testMappingVersionAfterDynamicMappingUpdate() throws Exception { createIndex("test"); final ClusterService clusterService = internalCluster().clusterService(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 9b6353b862b24..da4b8673c362b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -277,8 +277,14 @@ final ObjectMapper getDynamicObjectMapper(String name) { /** * Add a new runtime field dynamically created while parsing. + * We use the same set for both new indexed and new runtime fields, + * because for dynamic mappings, a new field can be either mapped + * as runtime or indexed, but never both. */ - public final void addDynamicRuntimeField(RuntimeField runtimeField) { + final void addDynamicRuntimeField(RuntimeField runtimeField) { + if (newFieldsSeen.add(runtimeField.name())) { + mappingLookup.checkFieldLimit(indexSettings().getMappingTotalFieldsLimit(), newFieldsSeen.size()); + } dynamicRuntimeFields.add(runtimeField); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java index 19f02f9ad0b84..a45fa7ff0e248 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -47,6 +47,7 @@ private CacheKey() {} /** Full field name to mapper */ private final Map fieldMappers; private final Map objectMappers; + private final int runtimeFieldMappersCount; private final NestedLookup nestedLookup; private final FieldTypeLookup fieldTypeLookup; private final FieldTypeLookup indexTimeLookup; // for index-time scripts, a lookup that does not include runtime fields @@ -180,6 +181,7 @@ private MappingLookup( // make all fields into compact+fast immutable maps this.fieldMappers = Map.copyOf(fieldMappers); this.objectMappers = Map.copyOf(objects); + this.runtimeFieldMappersCount = runtimeFields.size(); this.indexAnalyzersMap = Map.copyOf(indexAnalyzersMap); this.completionFields = Set.copyOf(completionFields); this.indexTimeScriptMappers = List.copyOf(indexTimeScriptMappers); @@ -262,7 +264,8 @@ private void checkFieldLimit(long limit) { } void checkFieldLimit(long limit, int additionalFieldsToAdd) { - if (fieldMappers.size() + objectMappers.size() + additionalFieldsToAdd - mapping.getSortedMetadataMappers().length > limit) { + if (fieldMappers.size() + objectMappers.size() + runtimeFieldMappersCount + additionalFieldsToAdd - mapping + .getSortedMetadataMappers().length > limit) { throw new IllegalArgumentException( "Limit of total fields [" + limit diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index f3771510d8da9..38d7567ce40e3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -70,6 +70,13 @@ public void testTotalFieldsLimit() throws Throwable { () -> merge(mapperService, mapping(b -> b.startObject("newfield").field("type", "long").endObject())) ); assertTrue(e.getMessage(), e.getMessage().contains("Limit of total fields [" + totalFieldsLimit + "] has been exceeded")); + + // adding one more runtime field should trigger exception + e = expectThrows( + IllegalArgumentException.class, + () -> merge(mapperService, runtimeMapping(b -> b.startObject("newfield").field("type", "long").endObject())) + ); + assertTrue(e.getMessage(), e.getMessage().contains("Limit of total fields [" + totalFieldsLimit + "] has been exceeded")); } private void createMappingSpecifyingNumberOfFields(XContentBuilder b, int numberOfFields) throws IOException {