Skip to content

Commit

Permalink
Preprocess mapping lookup to avoid copy_to fields during document par…
Browse files Browse the repository at this point in the history
…sing
  • Loading branch information
carlosdelest committed Apr 22, 2024
1 parent c2ab40f commit 3127da2
Show file tree
Hide file tree
Showing 3 changed files with 41 additions and 6 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -396,7 +396,7 @@ static void parseObjectOrField(DocumentParserContext context, Mapper mapper) thr
fieldMapper.parse(context);
}
if (context.isWithinCopyTo() == false) {
List<String> copyToFields = fieldMapper.copyTo().copyToFields();
List<String> copyToFields = context.mappingLookup().copyToFieldsForParsing(mapper.name());
if (copyToFields.isEmpty() == false) {
XContentParser.Token currentToken = context.parser().currentToken();
if (currentToken.isValue() == false && currentToken != XContentParser.Token.VALUE_NULL) {
Expand Down Expand Up @@ -696,10 +696,6 @@ private static void failIfMatchesRoutingPath(DocumentParserContext context, Stri
*/
private static void parseCopyFields(DocumentParserContext context, List<String> copyToFields) throws IOException {
for (String field : copyToFields) {
if (context.mappingLookup().inferenceFields().get(field) != null) {
// ignore copy_to that targets inference fields, values are already extracted in the coordinating node to perform inference.
continue;
}
// In case of a hierarchy of nested documents, we need to figure out
// which document the field should go to
LuceneDocument targetDoc = null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
Expand All @@ -36,6 +37,12 @@ final class FieldTypeLookup {
*/
private final Map<String, Set<String>> fieldToCopiedFields;

/**
* Contains the effective copy_to fields for each field. Fields that don't have
* to be actually parsed by the target copy_to fields will be removed from this list.
*/
private final Map<String, List<String>> copyToFields;

private final int maxParentPathDots;

FieldTypeLookup(
Expand All @@ -48,6 +55,8 @@ final class FieldTypeLookup {
final Map<String, String> fullSubfieldNameToParentPath = new HashMap<>();
final Map<String, DynamicFieldType> dynamicFieldTypes = new HashMap<>();
final Map<String, Set<String>> fieldToCopiedFields = new HashMap<>();
final Map<String, List<String>> effectiveCopyTo = new HashMap<>();
final Set<String> inferenceFieldMappers = new HashSet<>();
for (FieldMapper fieldMapper : fieldMappers) {
String fieldName = fieldMapper.name();
MappedFieldType fieldType = fieldMapper.fieldType();
Expand All @@ -56,7 +65,9 @@ final class FieldTypeLookup {
if (fieldType instanceof DynamicFieldType) {
dynamicFieldTypes.put(fieldType.name(), (DynamicFieldType) fieldType);
}
for (String targetField : fieldMapper.copyTo().copyToFields()) {
List<String> copyToFields = fieldMapper.copyTo().copyToFields();
effectiveCopyTo.put(fieldName, copyToFields);
for (String targetField : copyToFields) {
Set<String> sourcePath = fieldToCopiedFields.get(targetField);
if (sourcePath == null) {
Set<String> copiedFields = new HashSet<>();
Expand All @@ -65,6 +76,14 @@ final class FieldTypeLookup {
}
fieldToCopiedFields.get(targetField).add(fieldName);
}
if (fieldMapper instanceof InferenceFieldMapper) {
inferenceFieldMappers.add(fieldName);
}
}

// Remove all inference field mappers from the effective copy to list
for (FieldMapper fieldMapper : fieldMappers) {
effectiveCopyTo.get(fieldMapper.name()).removeAll(inferenceFieldMappers);
}

int maxParentPathDots = 0;
Expand Down Expand Up @@ -97,6 +116,8 @@ final class FieldTypeLookup {
// make values into more compact immutable sets to save memory
fieldToCopiedFields.entrySet().forEach(e -> e.setValue(Set.copyOf(e.getValue())));
this.fieldToCopiedFields = Map.copyOf(fieldToCopiedFields);
effectiveCopyTo.entrySet().forEach(e -> e.setValue(List.copyOf(e.getValue())));
this.copyToFields = Map.copyOf(effectiveCopyTo);
}

public static int dotCount(String path) {
Expand Down Expand Up @@ -205,6 +226,10 @@ Set<String> sourcePaths(String field) {
return fieldToCopiedFields.containsKey(resolvedField) ? fieldToCopiedFields.get(resolvedField) : Set.of(resolvedField);
}

List<String> copyToFields(String field) {
return copyToFields.getOrDefault(field, Collections.emptyList());
}

/**
* If field is a leaf multi-field return the path to the parent field. Otherwise, return null.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,7 @@ private MappingLookup(
this.mapping = mapping;
Map<String, Mapper> fieldMappers = new HashMap<>();
Map<String, ObjectMapper> objects = new HashMap<>();
Map<String, List<String>> effectiveCopyTo = new HashMap<>();

List<NestedObjectMapper> nestedMappers = new ArrayList<>();
for (ObjectMapper mapper : objectMappers) {
Expand Down Expand Up @@ -448,6 +449,19 @@ public Set<String> sourcePaths(String field) {
return fieldTypesLookup().sourcePaths(field);
}

/**
* Retrieves the effective copy to fields for the given field that should be parsed. Some fields may skip
* parsing as the value may have already been calculated by another process and doesn't need to be parsed
* by the document parser
*
* @param field the field for which to look up the _source path. Note that the field. Note that the field should be a
* concrete field and *not* an alias.
* @return A list of field names that the field should be copied to for parsing
*/
public List<String> copyToFieldsForParsing(String field) {
return fieldTypesLookup().copyToFields(field);
}

/**
* If field is a leaf multi-field return the path to the parent field. Otherwise, return null.
*/
Expand Down

0 comments on commit 3127da2

Please sign in to comment.