Skip to content

Commit

Permalink
Don't require DocumentMapper as an argument when parsing a document (#…
Browse files Browse the repository at this point in the history 
…66780)

Currently, an incoming document is parsed through `DocumentMapper#parse`, which in turns calls `DocumentParser#parseDocument` providing `this` among other arguments. As part of the effort to reduce usages of `DocumentMapper` when possible, as it represents the mutable side of mappings (through mappings updates) and involves complexity, we can carry around only the needed components. This does add some required arguments to `DocumentParser#parseDocument` , though it makes dependencies clearer. This change does not affect end consumers as they all go through DocumentMapper anyways, but by not needed to provide DocumentMapper to parseDocument, we may be able to unblock further improvements down the line.

Relates to #66295
  • Loading branch information
@javanna
javanna authored Jan 5, 2021
1 parent dad3396 commit b29c09b
Show file tree
Hide file tree
Showing 6 changed files with 66 additions and 45 deletions.
3 changes: 2 additions & 1 deletion modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1119,7 +1119,8 @@ private void duelRun(PercolateQuery.QueryStore queryStore, MemoryIndex memoryInd
}

private void addQuery(Query query, List<ParseContext.Document> docs) {
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper.mapping(),
documentMapper.mappers(), mapperService.getIndexSettings(), mapperService.getIndexAnalyzers(), null, null, null, null);
fieldMapper.processQuery(query, parseContext);
ParseContext.Document queryDocument = parseContext.doc();
// Add to string representation of the query to make debugging easier:
Expand Down
18 changes: 12 additions & 6 deletions ...les/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,8 @@ public void testExtractTerms() throws Exception {

DocumentMapper documentMapper = mapperService.documentMapper("doc");
PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName);
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper.mapping(),
documentMapper.mappers(), mapperService.getIndexSettings(), mapperService.getIndexAnalyzers(), null, null, null, null);
fieldMapper.processQuery(bq.build(), parseContext);
ParseContext.Document document = parseContext.doc();

Expand All @@ -202,7 +203,8 @@ public void testExtractTerms() throws Exception {
bq.add(termQuery1, Occur.MUST);
bq.add(termQuery2, Occur.MUST);

parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
parseContext = new ParseContext.InternalParseContext(documentMapper.mapping(),
documentMapper.mappers(), mapperService.getIndexSettings(), mapperService.getIndexAnalyzers(), null, null, null, null);
fieldMapper.processQuery(bq.build(), parseContext);
document = parseContext.doc();

Expand Down Expand Up @@ -231,7 +233,8 @@ public void testExtractRanges() throws Exception {

DocumentMapper documentMapper = mapperService.documentMapper("doc");
PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName);
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper.mapping(),
documentMapper.mappers(), mapperService.getIndexSettings(), mapperService.getIndexAnalyzers(), null, null, null, null);
fieldMapper.processQuery(bq.build(), parseContext);
ParseContext.Document document = parseContext.doc();

Expand All @@ -256,7 +259,8 @@ public void testExtractRanges() throws Exception {
.rangeQuery(15, 20, true, true, null, null, null, context);
bq.add(rangeQuery2, Occur.MUST);

parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
parseContext = new ParseContext.InternalParseContext(documentMapper.mapping(),
documentMapper.mappers(), mapperService.getIndexSettings(), mapperService.getIndexAnalyzers(), null, null, null, null);
fieldMapper.processQuery(bq.build(), parseContext);
document = parseContext.doc();

Expand All @@ -279,7 +283,8 @@ public void testExtractTermsAndRanges_failed() throws Exception {
TermRangeQuery query = new TermRangeQuery("field1", new BytesRef("a"), new BytesRef("z"), true, true);
DocumentMapper documentMapper = mapperService.documentMapper("doc");
PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName);
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper.mapping(),
documentMapper.mappers(), mapperService.getIndexSettings(), mapperService.getIndexAnalyzers(), null, null, null, null);
fieldMapper.processQuery(query, parseContext);
ParseContext.Document document = parseContext.doc();

Expand All @@ -293,7 +298,8 @@ public void testExtractTermsAndRanges_partial() throws Exception {
PhraseQuery phraseQuery = new PhraseQuery("field", "term");
DocumentMapper documentMapper = mapperService.documentMapper("doc");
PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName);
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper.mapping(),
documentMapper.mappers(), mapperService.getIndexSettings(), mapperService.getIndexAnalyzers(), null, null, null, null);
fieldMapper.processQuery(phraseQuery, parseContext);
ParseContext.Document document = parseContext.doc();

Expand Down
17 changes: 5 additions & 12 deletions server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@
import java.util.function.Function;
import java.util.stream.Stream;


public class DocumentMapper implements ToXContentFragment {

public static class Builder {
Expand Down Expand Up @@ -134,14 +133,6 @@ private DocumentMapper(IndexSettings indexSettings,
.filter(field -> noopTombstoneMetadataFields.contains(field.name())).toArray(MetadataFieldMapper[]::new);
}

IndexSettings indexSettings() {
return indexSettings;
}

IndexAnalyzers indexAnalyzers() {
return indexAnalyzers;
}

public Mapping mapping() {
return mapping;
}
Expand Down Expand Up @@ -203,18 +194,20 @@ public MappingLookup mappers() {
}

public ParsedDocument parse(SourceToParse source) throws MapperParsingException {
return documentParser.parseDocument(source, mapping.metadataMappers, this);
return documentParser.parseDocument(source, mapping, fieldMappers, indexSettings, indexAnalyzers);
}

public ParsedDocument createDeleteTombstoneDoc(String index, String type, String id) throws MapperParsingException {
final SourceToParse emptySource = new SourceToParse(index, type, id, new BytesArray("{}"), XContentType.JSON);
return documentParser.parseDocument(emptySource, deleteTombstoneMetadataFieldMappers, this).toTombstone();
return documentParser.parseDocument(emptySource, mapping, deleteTombstoneMetadataFieldMappers, fieldMappers,
indexSettings, indexAnalyzers).toTombstone();
}

public ParsedDocument createNoopTombstoneDoc(String index, String reason) throws MapperParsingException {
final String id = ""; // _id won't be used.
final SourceToParse sourceToParse = new SourceToParse(index, type, id, new BytesArray("{}"), XContentType.JSON);
final ParsedDocument parsedDoc = documentParser.parseDocument(sourceToParse, noopTombstoneMetadataFieldMappers, this).toTombstone();
final ParsedDocument parsedDoc = documentParser.parseDocument(sourceToParse, mapping, noopTombstoneMetadataFieldMappers,
fieldMappers, indexSettings, indexAnalyzers).toTombstone();
// Store the reason of a noop as a raw string in the _source field
final BytesRef byteRef = new BytesRef(reason);
parsedDoc.rootDoc().add(new StoredField(SourceFieldMapper.NAME, byteRef.bytes, byteRef.offset, byteRef.length));
Expand Down
54 changes: 35 additions & 19 deletions server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.query.QueryShardContext;

/** A parser for documents, given mappings from a DocumentMapper */
Expand All @@ -58,19 +60,34 @@ final class DocumentParser {
}

ParsedDocument parseDocument(SourceToParse source,
MetadataFieldMapper[] metadataFieldsMappers,
DocumentMapper docMapper) throws MapperParsingException {
validateType(source, docMapper);
Mapping mapping,
MappingLookup mappingLookup,
IndexSettings indexSettings,
IndexAnalyzers indexAnalyzers) throws MapperParsingException {
return parseDocument(source, mapping, mapping.metadataMappers, mappingLookup, indexSettings, indexAnalyzers);
}

final Mapping mapping = docMapper.mapping();
ParsedDocument parseDocument(SourceToParse source,
Mapping mapping,
MetadataFieldMapper[] metadataFieldsMappers,
MappingLookup mappingLookup,
IndexSettings indexSettings,
IndexAnalyzers indexAnalyzers) throws MapperParsingException {
validateType(source, mappingLookup.getType());
final ParseContext.InternalParseContext context;
final XContentType xContentType = source.getXContentType();

try (XContentParser parser = XContentHelper.createParser(xContentRegistry,
LoggingDeprecationHandler.INSTANCE, source.source(), xContentType)) {
context = new ParseContext.InternalParseContext(docMapper, dateParserContext, dynamicRuntimeFieldsBuilder, source, parser);
context = new ParseContext.InternalParseContext(mapping,
mappingLookup,
indexSettings,
indexAnalyzers,
dateParserContext,
dynamicRuntimeFieldsBuilder,
source,
parser);
validateStart(parser);
internalParseDocument(mapping, metadataFieldsMappers, context, parser);
internalParseDocument(mapping.root(), metadataFieldsMappers, context, parser);
validateEnd(parser);
} catch (Exception e) {
throw wrapInMapperParsingException(source, e);
Expand All @@ -82,7 +99,7 @@ ParsedDocument parseDocument(SourceToParse source,

context.postParse();

return parsedDocument(source, context, createDynamicUpdate(mapping, docMapper.mappers(),
return parsedDocument(source, context, createDynamicUpdate(mapping, mappingLookup,
context.getDynamicMappers(), context.getDynamicRuntimeFields()));
}

Expand All @@ -100,35 +117,34 @@ private static boolean containsDisabledObjectMapper(ObjectMapper objectMapper, S
return false;
}

private static void internalParseDocument(Mapping mapping, MetadataFieldMapper[] metadataFieldsMappers,
private static void internalParseDocument(RootObjectMapper root, MetadataFieldMapper[] metadataFieldsMappers,
ParseContext context, XContentParser parser) throws IOException {
final boolean emptyDoc = isEmptyDoc(mapping, parser);
final boolean emptyDoc = isEmptyDoc(root, parser);

for (MetadataFieldMapper metadataMapper : metadataFieldsMappers) {
metadataMapper.preParse(context);
}

if (mapping.root.isEnabled() == false) {
if (root.isEnabled() == false) {
// entire type is disabled
parser.skipChildren();
} else if (emptyDoc == false) {
parseObjectOrNested(context, mapping.root);
parseObjectOrNested(context, root);
}

for (MetadataFieldMapper metadataMapper : metadataFieldsMappers) {
metadataMapper.postParse(context);
}
}

private void validateType(SourceToParse source, DocumentMapper docMapper) {
if (docMapper.type().equals(MapperService.DEFAULT_MAPPING)) {
private void validateType(SourceToParse source, String type) {
if (type.equals(MapperService.DEFAULT_MAPPING)) {
throw new IllegalArgumentException("It is forbidden to index into the default mapping [" + MapperService.DEFAULT_MAPPING + "]");
}

if (Objects.equals(source.type(), docMapper.type()) == false &&
if (Objects.equals(source.type(), type) == false &&
MapperService.SINGLE_MAPPING_NAME.equals(source.type()) == false) { // used by typeless APIs
throw new MapperParsingException("Type mismatch, provide type [" + source.type() + "] but mapper is of type ["
+ docMapper.type() + "]");
throw new MapperParsingException("Type mismatch, provide type [" + source.type() + "] but mapper is of type [" + type + "]");
}
}

Expand All @@ -150,8 +166,8 @@ private static void validateEnd(XContentParser parser) throws IOException {
}
}

private static boolean isEmptyDoc(Mapping mapping, XContentParser parser) throws IOException {
if (mapping.root.isEnabled()) {
private static boolean isEmptyDoc(RootObjectMapper root, XContentParser parser) throws IOException {
if (root.isEnabled()) {
final XContentParser.Token token = parser.nextToken();
if (token == XContentParser.Token.END_OBJECT) {
// empty doc, we can handle it...
Expand Down
15 changes: 9 additions & 6 deletions server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -342,23 +342,26 @@ public static class InternalParseContext extends ParseContext {
private long numNestedDocs;
private boolean docsReversed = false;

public InternalParseContext(DocumentMapper docMapper,
public InternalParseContext(Mapping mapping,
MappingLookup mappingLookup,
IndexSettings indexSettings,
IndexAnalyzers indexAnalyzers,
Function<DateFormatter, Mapper.TypeParser.ParserContext> parserContextFunction,
DynamicRuntimeFieldsBuilder dynamicRuntimeFieldsBuilder,
SourceToParse source,
XContentParser parser) {
this.mapping = docMapper.mapping();
this.mappingLookup = docMapper.mappers();
this.indexSettings = docMapper.indexSettings();
this.indexAnalyzers = docMapper.indexAnalyzers();
this.mapping = mapping;
this.mappingLookup = mappingLookup;
this.indexSettings = indexSettings;
this.indexAnalyzers = indexAnalyzers;
this.parserContextFunction = parserContextFunction;
this.dynamicRuntimeFieldsBuilder = dynamicRuntimeFieldsBuilder;
this.parser = parser;
this.document = new Document();
this.documents.add(document);
this.version = null;
this.sourceToParse = source;
this.maxAllowedNumNestedDocs = docMapper.indexSettings().getMappingNestedDocsLimit();
this.maxAllowedNumNestedDocs = indexSettings.getMappingNestedDocsLimit();
this.numNestedDocs = 0L;
}

Expand Down
4 changes: 3 additions & 1 deletion server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -617,7 +617,9 @@ MapperService createMapperService() throws Exception {

// creates an object mapper, which is about 100x harder than it should be....
ObjectMapper createObjectMapper(MapperService mapperService, String name) {
ParseContext context = new ParseContext.InternalParseContext(mapperService.documentMapper(), null, null, null, null);
DocumentMapper docMapper = mapperService.documentMapper();
ParseContext context = new ParseContext.InternalParseContext(docMapper.mapping(), docMapper.mappers(),
mapperService.getIndexSettings(), mapperService.getIndexAnalyzers(), null, null, null, null);
String[] nameParts = name.split("\\.");
for (int i = 0; i < nameParts.length - 1; ++i) {
context.path().add(nameParts[i]);
Expand Down

0 comments on commit b29c09b

Please sign in to comment.