Skip to content

Commit

Permalink
Store parsed mapping settings in IndexSettings (#57492)
Browse files Browse the repository at this point in the history
There are several mapping settings that are currently re-parsed every
time they are read. This can be quite frequent, for example within every
document ingestion. This commit moves the parsed versions of these
mapping settings to be stored in IndexSettings, just as other index settings
are already.

closes #57395
  • Loading branch information
rjernst committed Jun 1, 2020
1 parent db5bf92 commit 7aad4f6
Show file tree
Hide file tree
Showing 3 changed files with 66 additions and 5 deletions.
61 changes: 61 additions & 0 deletions server/src/main/java/org/elasticsearch/index/IndexSettings.java
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,12 @@
import java.util.function.Consumer;
import java.util.function.Function;

import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING;
import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING;
import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING;
import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING;
import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING;

/**
* This class encapsulates all index level settings and handles settings updates.
* It's created per index and available to all index level classes and allows them to retrieve
Expand Down Expand Up @@ -402,6 +408,11 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) {
private volatile String defaultPipeline;
private volatile String requiredPipeline;
private volatile boolean searchThrottled;
private volatile long mappingNestedFieldsLimit;
private volatile long mappingNestedDocsLimit;
private volatile long mappingTotalFieldsLimit;
private volatile long mappingDepthLimit;
private volatile long mappingFieldNameLengthLimit;

/**
* The maximum number of refresh listeners allows on this shard.
Expand Down Expand Up @@ -523,6 +534,11 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
defaultPipeline = scopedSettings.get(DEFAULT_PIPELINE);
setTranslogRetentionAge(scopedSettings.get(INDEX_TRANSLOG_RETENTION_AGE_SETTING));
setTranslogRetentionSize(scopedSettings.get(INDEX_TRANSLOG_RETENTION_SIZE_SETTING));
mappingNestedFieldsLimit = scopedSettings.get(INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING);
mappingNestedDocsLimit = scopedSettings.get(INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING);
mappingTotalFieldsLimit = scopedSettings.get(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING);
mappingDepthLimit = scopedSettings.get(INDEX_MAPPING_DEPTH_LIMIT_SETTING);
mappingFieldNameLengthLimit = scopedSettings.get(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING);

scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, mergePolicyConfig::setNoCFSRatio);
scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING,
Expand Down Expand Up @@ -576,6 +592,11 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
scopedSettings.addSettingsUpdateConsumer(INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING, this::setSoftDeleteRetentionOperations);
scopedSettings.addSettingsUpdateConsumer(INDEX_SEARCH_THROTTLED, this::setSearchThrottled);
scopedSettings.addSettingsUpdateConsumer(INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING, this::setRetentionLeaseMillis);
scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING, this::setMappingNestedFieldsLimit);
scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING, this::setMappingNestedDocsLimit);
scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING, this::setMappingTotalFieldsLimit);
scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DEPTH_LIMIT_SETTING, this::setMappingDepthLimit);
scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, this::setMappingFieldNameLengthLimit);
}

private void setSearchIdleAfter(TimeValue searchIdleAfter) { this.searchIdleAfter = searchIdleAfter; }
Expand Down Expand Up @@ -1065,4 +1086,44 @@ public boolean isSearchThrottled() {
private void setSearchThrottled(boolean searchThrottled) {
this.searchThrottled = searchThrottled;
}

public long getMappingNestedFieldsLimit() {
return mappingNestedFieldsLimit;
}

private void setMappingNestedFieldsLimit(long value) {
this.mappingNestedFieldsLimit = value;
}

public long getMappingNestedDocsLimit() {
return mappingNestedDocsLimit;
}

private void setMappingNestedDocsLimit(long value) {
this.mappingNestedDocsLimit = value;
}

public long getMappingTotalFieldsLimit() {
return mappingTotalFieldsLimit;
}

private void setMappingTotalFieldsLimit(long value) {
this.mappingTotalFieldsLimit = value;
}

public long getMappingDepthLimit() {
return mappingDepthLimit;
}

private void setMappingDepthLimit(long value) {
this.mappingDepthLimit = value;
}

public long getMappingFieldNameLengthLimit() {
return mappingFieldNameLengthLimit;
}

private void setMappingFieldNameLengthLimit(long value) {
this.mappingFieldNameLengthLimit = value;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -574,7 +574,7 @@ private boolean assertSerialization(DocumentMapper mapper) {
}

private void checkNestedFieldsLimit(Map<String, ObjectMapper> fullPathObjectMappers) {
long allowedNestedFields = indexSettings.getValue(INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING);
long allowedNestedFields = indexSettings.getMappingNestedFieldsLimit();
long actualNestedFields = 0;
for (ObjectMapper objectMapper : fullPathObjectMappers.values()) {
if (objectMapper.nested().isNested()) {
Expand All @@ -588,15 +588,15 @@ private void checkNestedFieldsLimit(Map<String, ObjectMapper> fullPathObjectMapp
}

private void checkTotalFieldsLimit(long totalMappers) {
long allowedTotalFields = indexSettings.getValue(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING);
long allowedTotalFields = indexSettings.getMappingTotalFieldsLimit();
if (allowedTotalFields < totalMappers) {
throw new IllegalArgumentException("Limit of total fields [" + allowedTotalFields + "] in index [" + index().getName()
+ "] has been exceeded");
}
}

private void checkDepthLimit(Collection<String> objectPaths) {
final long maxDepth = indexSettings.getValue(INDEX_MAPPING_DEPTH_LIMIT_SETTING);
final long maxDepth = indexSettings.getMappingDepthLimit();
for (String objectPath : objectPaths) {
checkDepthLimit(objectPath, maxDepth);
}
Expand All @@ -619,7 +619,7 @@ private void checkDepthLimit(String objectPath, long maxDepth) {
private void checkFieldNameSoftLimit(Collection<ObjectMapper> objectMappers,
Collection<FieldMapper> fieldMappers,
Collection<FieldAliasMapper> fieldAliasMappers) {
final long maxFieldNameLength = indexSettings.getValue(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING);
final long maxFieldNameLength = indexSettings.getMappingFieldNameLengthLimit();

Stream.of(objectMappers.stream(), fieldMappers.stream(), fieldAliasMappers.stream())
.reduce(Stream::concat)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -333,7 +333,7 @@ public InternalParseContext(IndexSettings indexSettings, DocumentMapperParser do
this.version = null;
this.sourceToParse = source;
this.dynamicMappers = new ArrayList<>();
this.maxAllowedNumNestedDocs = indexSettings.getValue(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING);
this.maxAllowedNumNestedDocs = indexSettings.getMappingNestedDocsLimit();
this.numNestedDocs = 0L;
}

Expand Down

0 comments on commit 7aad4f6

Please sign in to comment.