diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java index 1ad4aff24977b..f129723448456 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java @@ -91,7 +91,6 @@ public static final class RankFeatureFieldType extends MappedFieldType { public RankFeatureFieldType(String name, Map meta, boolean positiveScoreImpact) { super(name, true, false, false, TextSearchInfo.NONE, meta); this.positiveScoreImpact = positiveScoreImpact; - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); } @Override @@ -136,7 +135,7 @@ public Query termQuery(Object value, QueryShardContext context) { private RankFeatureFieldMapper(String simpleName, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo, boolean positiveScoreImpact) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, multiFields, copyTo); this.positiveScoreImpact = positiveScoreImpact; } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java index 780aa26f742dc..3bf1877461c75 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java @@ -68,7 +68,6 @@ public static final class RankFeaturesFieldType extends MappedFieldType { public RankFeaturesFieldType(String name, Map meta) { super(name, false, false, false, TextSearchInfo.NONE, meta); - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); } @Override @@ -99,7 +98,7 @@ public Query termQuery(Object value, QueryShardContext context) { private RankFeaturesFieldMapper(String simpleName, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, multiFields, copyTo); } @Override diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java index ae1ceca69fc56..1124caedccbd7 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -57,6 +57,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -95,18 +96,14 @@ public static class Defaults { public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, () -> c.getIndexAnalyzers().getDefaultIndexAnalyzer())); - private static SearchAsYouTypeFieldMapper toType(FieldMapper in) { - return (SearchAsYouTypeFieldMapper) in; - } - - private static SearchAsYouTypeFieldType ft(FieldMapper in) { - return toType(in).fieldType(); + private static Builder builder(FieldMapper in) { + return ((SearchAsYouTypeFieldMapper)in).builder; } public static class Builder extends FieldMapper.Builder { - private final Parameter index = Parameter.indexParam(m -> toType(m).index, true); - private final Parameter store = Parameter.storeParam(m -> toType(m).store, false); + private final Parameter index = Parameter.indexParam(m -> builder(m).index.get(), true); + private final Parameter store = Parameter.storeParam(m -> builder(m).store.get(), false); // This is only here because for some reason the initial impl of this always serialized // `doc_values=false`, even though it cannot be set; and so we need to continue @@ -120,7 +117,7 @@ public static class Builder extends FieldMapper.Builder { .alwaysSerialize(); private final Parameter maxShingleSize = Parameter.intParam("max_shingle_size", false, - m -> toType(m).maxShingleSize, Defaults.MAX_SHINGLE_SIZE) + m -> builder(m).maxShingleSize.get(), Defaults.MAX_SHINGLE_SIZE) .setValidator(v -> { if (v < MAX_SHINGLE_SIZE_LOWER_BOUND || v > MAX_SHINGLE_SIZE_UPPER_BOUND) { throw new MapperParsingException("[max_shingle_size] must be at least [" + MAX_SHINGLE_SIZE_LOWER_BOUND @@ -130,17 +127,17 @@ public static class Builder extends FieldMapper.Builder { .alwaysSerialize(); final TextParams.Analyzers analyzers; - final Parameter similarity = TextParams.similarity(m -> ft(m).getTextSearchInfo().getSimilarity()); + final Parameter similarity = TextParams.similarity(m -> builder(m).similarity.get()); - final Parameter indexOptions = TextParams.indexOptions(m -> toType(m).indexOptions); - final Parameter norms = TextParams.norms(true, m -> ft(m).getTextSearchInfo().hasNorms()); - final Parameter termVectors = TextParams.termVectors(m -> toType(m).termVectors); + final Parameter indexOptions = TextParams.indexOptions(m -> builder(m).indexOptions.get()); + final Parameter norms = TextParams.norms(true, m -> builder(m).norms.get()); + final Parameter termVectors = TextParams.termVectors(m -> builder(m).termVectors.get()); private final Parameter> meta = Parameter.metaParam(); public Builder(String name, Supplier defaultAnalyzer) { super(name); - this.analyzers = new TextParams.Analyzers(defaultAnalyzer); + this.analyzers = new TextParams.Analyzers(defaultAnalyzer, m -> builder(m).analyzers); } @Override @@ -159,12 +156,15 @@ public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) { fieldType.setStored(store.getValue()); TextParams.setTermVectorParams(termVectors.getValue(), fieldType); + Map indexAnalyzers = new HashMap<>(); + NamedAnalyzer indexAnalyzer = analyzers.getIndexAnalyzer(); NamedAnalyzer searchAnalyzer = analyzers.getSearchAnalyzer(); SearchAsYouTypeFieldType ft = new SearchAsYouTypeFieldType(buildFullName(context), fieldType, similarity.getValue(), analyzers.getSearchAnalyzer(), analyzers.getSearchQuoteAnalyzer(), meta.getValue()); - ft.setIndexAnalyzer(analyzers.getIndexAnalyzer()); + + indexAnalyzers.put(ft.name(), indexAnalyzer); // set up the prefix field FieldType prefixft = new FieldType(fieldType); @@ -182,8 +182,10 @@ public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) { TextSearchInfo prefixSearchInfo = new TextSearchInfo(prefixft, similarity.getValue(), prefixSearchWrapper, searchAnalyzer); final PrefixFieldType prefixFieldType = new PrefixFieldType(fullName, prefixSearchInfo, Defaults.MIN_GRAM, Defaults.MAX_GRAM); - prefixFieldType.setIndexAnalyzer(new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX, prefixIndexWrapper)); + final NamedAnalyzer prefixAnalyzer + = new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX, prefixIndexWrapper); final PrefixFieldMapper prefixFieldMapper = new PrefixFieldMapper(prefixft, prefixFieldType); + indexAnalyzers.put(prefixFieldType.name(), prefixAnalyzer); // set up the shingle fields final ShingleFieldMapper[] shingleFieldMappers = new ShingleFieldMapper[maxShingleSize.getValue() - 1]; @@ -203,14 +205,16 @@ public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) { TextSearchInfo textSearchInfo = new TextSearchInfo(shingleft, similarity.getValue(), shingleSearchWrapper, shingleSearchQuoteWrapper); final ShingleFieldType shingleFieldType = new ShingleFieldType(fieldName, shingleSize, textSearchInfo); - shingleFieldType.setIndexAnalyzer(new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX, shingleIndexWrapper)); shingleFieldType.setPrefixFieldType(prefixFieldType); shingleFieldTypes[i] = shingleFieldType; + NamedAnalyzer shingleAnalyzer + = new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX, shingleIndexWrapper); shingleFieldMappers[i] = new ShingleFieldMapper(shingleft, shingleFieldType); + indexAnalyzers.put(shingleFieldType.name(), shingleAnalyzer); } ft.setPrefixField(prefixFieldType); ft.setShingleFields(shingleFieldTypes); - return new SearchAsYouTypeFieldMapper(name, ft, copyTo.build(), prefixFieldMapper, shingleFieldMappers, this); + return new SearchAsYouTypeFieldMapper(name, ft, copyTo.build(), indexAnalyzers, prefixFieldMapper, shingleFieldMappers, this); } } @@ -546,29 +550,23 @@ public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRew } } - private final boolean index; - private final boolean store; - private final String indexOptions; - private final String termVectors; - private final int maxShingleSize; private final PrefixFieldMapper prefixField; private final ShingleFieldMapper[] shingleFields; + private final Builder builder; public SearchAsYouTypeFieldMapper(String simpleName, SearchAsYouTypeFieldType mappedFieldType, CopyTo copyTo, + Map indexAnalyzers, PrefixFieldMapper prefixField, ShingleFieldMapper[] shingleFields, Builder builder) { - super(simpleName, mappedFieldType, MultiFields.empty(), copyTo); + super(simpleName, mappedFieldType, indexAnalyzers, MultiFields.empty(), copyTo); this.prefixField = prefixField; this.shingleFields = shingleFields; this.maxShingleSize = builder.maxShingleSize.getValue(); - this.index = builder.index.getValue(); - this.store = builder.store.getValue(); - this.indexOptions = builder.indexOptions.getValue(); - this.termVectors = builder.termVectors.getValue(); + this.builder = builder; } @Override @@ -578,12 +576,12 @@ protected void parseCreateField(ParseContext context) throws IOException { return; } - if (this.index == false && this.store == false) { + if (this.builder.index.get() == false && this.builder.store.get() == false) { return; } context.doc().add(new Field(fieldType().name(), value, fieldType().fieldType)); - if (this.index) { + if (this.builder.index.get()) { for (ShingleFieldMapper subFieldMapper : shingleFields) { context.doc().add(new Field(subFieldMapper.fieldType().name(), value, subFieldMapper.getLuceneFieldType())); } @@ -599,9 +597,8 @@ protected String contentType() { return CONTENT_TYPE; } - @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), () -> fieldType().indexAnalyzer()).init(this); + return new Builder(simpleName(), builder.analyzers.indexAnalyzer::getDefaultValue).init(this); } public static String getShingleFieldName(String parentField, int shingleSize) { diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java index 156e09c005343..d476951e36034 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java @@ -179,12 +179,12 @@ public void testDefaultConfiguration() throws IOException { assertRootFieldMapper(rootMapper, 3, "default"); PrefixFieldMapper prefixFieldMapper = getPrefixFieldMapper(defaultMapper, "field._index_prefix"); - assertPrefixFieldType(prefixFieldMapper.fieldType(), 3, "default"); + assertPrefixFieldType(prefixFieldMapper, rootMapper.indexAnalyzers(), 3, "default"); - assertShingleFieldType( - getShingleFieldMapper(defaultMapper, "field._2gram").fieldType(), 2, "default", prefixFieldMapper.fieldType()); - assertShingleFieldType( - getShingleFieldMapper(defaultMapper, "field._3gram").fieldType(), 3, "default", prefixFieldMapper.fieldType()); + assertShingleFieldType(getShingleFieldMapper(defaultMapper, "field._2gram"), + rootMapper.indexAnalyzers(), 2, "default", prefixFieldMapper.fieldType()); + assertShingleFieldType(getShingleFieldMapper(defaultMapper, "field._3gram"), + rootMapper.indexAnalyzers(), 3, "default", prefixFieldMapper.fieldType()); } public void testConfiguration() throws IOException { @@ -200,14 +200,14 @@ public void testConfiguration() throws IOException { assertRootFieldMapper(rootMapper, maxShingleSize, analyzerName); PrefixFieldMapper prefixFieldMapper = getPrefixFieldMapper(defaultMapper, "field._index_prefix"); - assertPrefixFieldType(prefixFieldMapper.fieldType(), maxShingleSize, analyzerName); + assertPrefixFieldType(prefixFieldMapper, rootMapper.indexAnalyzers(), maxShingleSize, analyzerName); - assertShingleFieldType( - getShingleFieldMapper(defaultMapper, "field._2gram").fieldType(), 2, analyzerName, prefixFieldMapper.fieldType()); - assertShingleFieldType( - getShingleFieldMapper(defaultMapper, "field._3gram").fieldType(), 3, analyzerName, prefixFieldMapper.fieldType()); - assertShingleFieldType( - getShingleFieldMapper(defaultMapper, "field._4gram").fieldType(), 4, analyzerName, prefixFieldMapper.fieldType()); + assertShingleFieldType(getShingleFieldMapper(defaultMapper, "field._2gram"), + rootMapper.indexAnalyzers(), 2, analyzerName, prefixFieldMapper.fieldType()); + assertShingleFieldType(getShingleFieldMapper(defaultMapper, "field._3gram"), + rootMapper.indexAnalyzers(), 3, analyzerName, prefixFieldMapper.fieldType()); + assertShingleFieldType(getShingleFieldMapper(defaultMapper, "field._4gram"), + rootMapper.indexAnalyzers(), 4, analyzerName, prefixFieldMapper.fieldType()); } public void testSimpleMerge() throws IOException { @@ -599,47 +599,53 @@ private static void assertRootFieldMapper(SearchAsYouTypeFieldMapper mapper, assertThat(mapper.maxShingleSize(), equalTo(maxShingleSize)); assertThat(mapper.fieldType(), notNullValue()); - assertSearchAsYouTypeFieldType(mapper.fieldType(), maxShingleSize, analyzerName, mapper.prefixField().fieldType()); + assertSearchAsYouTypeFieldType(mapper, mapper.fieldType(), maxShingleSize, analyzerName, mapper.prefixField().fieldType()); assertThat(mapper.prefixField(), notNullValue()); assertThat(mapper.prefixField().fieldType().parentField, equalTo(mapper.name())); - assertPrefixFieldType(mapper.prefixField().fieldType(), maxShingleSize, analyzerName); + assertPrefixFieldType(mapper.prefixField(), mapper.indexAnalyzers, maxShingleSize, analyzerName); for (int shingleSize = 2; shingleSize <= maxShingleSize; shingleSize++) { final ShingleFieldMapper shingleFieldMapper = mapper.shingleFields()[shingleSize - 2]; assertThat(shingleFieldMapper, notNullValue()); - assertShingleFieldType(shingleFieldMapper.fieldType(), shingleSize, analyzerName, mapper.prefixField().fieldType()); + assertShingleFieldType(shingleFieldMapper, mapper.indexAnalyzers, shingleSize, + analyzerName, mapper.prefixField().fieldType()); } final int numberOfShingleSubfields = (maxShingleSize - 2) + 1; assertThat(mapper.shingleFields().length, equalTo(numberOfShingleSubfields)); } - private static void assertSearchAsYouTypeFieldType(SearchAsYouTypeFieldType fieldType, int maxShingleSize, + private static void assertSearchAsYouTypeFieldType(SearchAsYouTypeFieldMapper mapper, + SearchAsYouTypeFieldType fieldType, + int maxShingleSize, String analyzerName, PrefixFieldType prefixFieldType) { assertThat(fieldType.shingleFields.length, equalTo(maxShingleSize - 1)); - for (NamedAnalyzer analyzer : asList(fieldType.indexAnalyzer(), fieldType.getTextSearchInfo().getSearchAnalyzer())) { + NamedAnalyzer indexAnalyzer = mapper.indexAnalyzers().get(fieldType.name()); + for (NamedAnalyzer analyzer : asList(indexAnalyzer, fieldType.getTextSearchInfo().getSearchAnalyzer())) { assertThat(analyzer.name(), equalTo(analyzerName)); } int shingleSize = 2; - for (ShingleFieldType shingleField : fieldType.shingleFields) { - assertShingleFieldType(shingleField, shingleSize++, analyzerName, prefixFieldType); + for (ShingleFieldMapper shingleField : mapper.shingleFields()) { + assertShingleFieldType(shingleField, mapper.indexAnalyzers(), shingleSize++, analyzerName, prefixFieldType); } assertThat(fieldType.prefixField, equalTo(prefixFieldType)); } - private static void assertShingleFieldType(ShingleFieldType fieldType, + private static void assertShingleFieldType(ShingleFieldMapper mapper, + Map indexAnalyzers, int shingleSize, String analyzerName, PrefixFieldType prefixFieldType) { + ShingleFieldType fieldType = mapper.fieldType(); assertThat(fieldType.shingleSize, equalTo(shingleSize)); - for (NamedAnalyzer analyzer : asList(fieldType.indexAnalyzer(), fieldType.getTextSearchInfo().getSearchAnalyzer())) { + for (NamedAnalyzer analyzer : asList(indexAnalyzers.get(fieldType.name()), fieldType.getTextSearchInfo().getSearchAnalyzer())) { assertThat(analyzer.name(), equalTo(analyzerName)); if (shingleSize > 1) { final SearchAsYouTypeAnalyzer wrappedAnalyzer = (SearchAsYouTypeAnalyzer) analyzer.analyzer(); @@ -652,12 +658,15 @@ private static void assertShingleFieldType(ShingleFieldType fieldType, } - private static void assertPrefixFieldType(PrefixFieldType fieldType, int shingleSize, String analyzerName) { - for (NamedAnalyzer analyzer : asList(fieldType.indexAnalyzer(), fieldType.getTextSearchInfo().getSearchAnalyzer())) { + private static void assertPrefixFieldType(PrefixFieldMapper mapper, Map indexAnalyzers, + int shingleSize, String analyzerName) { + PrefixFieldType fieldType = mapper.fieldType(); + NamedAnalyzer indexAnalyzer = indexAnalyzers.get(fieldType.name()); + for (NamedAnalyzer analyzer : asList(indexAnalyzer, fieldType.getTextSearchInfo().getSearchAnalyzer())) { assertThat(analyzer.name(), equalTo(analyzerName)); } - final SearchAsYouTypeAnalyzer wrappedIndexAnalyzer = (SearchAsYouTypeAnalyzer) fieldType.indexAnalyzer().analyzer(); + final SearchAsYouTypeAnalyzer wrappedIndexAnalyzer = (SearchAsYouTypeAnalyzer) indexAnalyzer.analyzer(); final SearchAsYouTypeAnalyzer wrappedSearchAnalyzer = (SearchAsYouTypeAnalyzer) fieldType.getTextSearchInfo().getSearchAnalyzer().analyzer(); for (SearchAsYouTypeAnalyzer analyzer : asList(wrappedIndexAnalyzer, wrappedSearchAnalyzer)) { diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldTypeTests.java index dab80326afa18..31721fb77fe4a 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldTypeTests.java @@ -114,7 +114,6 @@ public void testPrefixQuery() { public void testFetchSourceValue() throws IOException { SearchAsYouTypeFieldType fieldType = createFieldType(); - fieldType.setIndexAnalyzer(Lucene.STANDARD_ANALYZER); assertEquals(List.of("value"), fetchSourceValue(fieldType, "value")); assertEquals(List.of("42"), fetchSourceValue(fieldType, 42L)); diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index 2392ef89e0069..9254c700b8c7c 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -60,7 +60,6 @@ static class Defaults { public static final class ParentIdFieldType extends StringFieldType { public ParentIdFieldType(String name, boolean eagerGlobalOrdinals) { super(name, true, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); setEagerGlobalOrdinals(eagerGlobalOrdinals); } @@ -91,7 +90,7 @@ public Object valueForDisplay(Object value) { } protected ParentIdFieldMapper(String name, boolean eagerGlobalOrdinals) { - super(name, new ParentIdFieldType(name, eagerGlobalOrdinals), MultiFields.empty(), CopyTo.empty()); + super(name, new ParentIdFieldType(name, eagerGlobalOrdinals), Lucene.KEYWORD_ANALYZER, MultiFields.empty(), CopyTo.empty()); } @Override diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index 68989bbbf60e9..9ff6c1682b603 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -144,7 +144,6 @@ public static final class JoinFieldType extends StringFieldType { private JoinFieldType(String name, Joiner joiner, Map meta) { super(name, true, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); this.joiner = joiner; } @@ -194,7 +193,7 @@ protected ParentJoinFieldMapper(String simpleName, MetaJoinFieldMapper uniqueFieldMapper, Map parentIdFields, boolean eagerGlobalOrdinals, List relations) { - super(simpleName, mappedFieldType, MultiFields.empty(), CopyTo.empty()); + super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, MultiFields.empty(), CopyTo.empty()); this.parentIdFields = parentIdFields; this.uniqueFieldMapper = uniqueFieldMapper; this.eagerGlobalOrdinals = eagerGlobalOrdinals; diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index ff961784fabda..51829cd6a71c0 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -62,7 +62,6 @@ public static final class CollationFieldType extends StringFieldType { public CollationFieldType(String name, boolean isSearchable, boolean isStored, boolean hasDocValues, Collator collator, String nullValue, int ignoreAbove, Map meta) { super(name, isSearchable, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); this.collator = collator; this.nullValue = nullValue; this.ignoreAbove = ignoreAbove; @@ -418,7 +417,7 @@ protected ICUCollationKeywordFieldMapper(String simpleName, FieldType fieldType, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo, Collator collator, Builder builder) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, multiFields, copyTo); assert collator.isFrozen(); this.fieldType = fieldType; this.params = builder.collatorParams(); diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index 9b1b36cf5129c..15e400ee7c2ef 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -35,7 +35,6 @@ import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TextParams; @@ -72,12 +71,16 @@ public class AnnotatedTextFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "annotated_text"; - private static final int POSITION_INCREMENT_GAP_USE_ANALYZER = -1; private static Builder builder(FieldMapper in) { return ((AnnotatedTextFieldMapper)in).builder; } + private static NamedAnalyzer wrapAnalyzer(NamedAnalyzer in) { + return new NamedAnalyzer(in.name(), AnalyzerScope.INDEX, + new AnnotationAnalyzerWrapper(in.analyzer()), in.getPositionIncrementGap("")); + } + public static class Builder extends FieldMapper.Builder { private final Parameter store = Parameter.storeParam(m -> builder(m).store.getValue(), false); @@ -90,55 +93,32 @@ public static class Builder extends FieldMapper.Builder { final Parameter norms = TextParams.norms(true, m -> builder(m).norms.getValue()); final Parameter termVectors = TextParams.termVectors(m -> builder(m).termVectors.getValue()); - final Parameter positionIncrementGap = Parameter.intParam("position_increment_gap", false, - m -> builder(m).positionIncrementGap.getValue(), POSITION_INCREMENT_GAP_USE_ANALYZER) - .setValidator(v -> { - if (v != POSITION_INCREMENT_GAP_USE_ANALYZER && v < 0) { - throw new MapperParsingException("[positions_increment_gap] must be positive, got [" + v + "]"); - } - }); - private final Parameter> meta = Parameter.metaParam(); public Builder(String name, Supplier defaultAnalyzer) { super(name); - this.analyzers = new TextParams.Analyzers(defaultAnalyzer); + this.analyzers = new TextParams.Analyzers(defaultAnalyzer, m -> builder(m).analyzers); } @Override protected List> getParameters() { return Arrays.asList(store, indexOptions, norms, termVectors, similarity, - analyzers.indexAnalyzer, analyzers.searchAnalyzer, analyzers.searchQuoteAnalyzer, positionIncrementGap, + analyzers.indexAnalyzer, analyzers.searchAnalyzer, analyzers.searchQuoteAnalyzer, + analyzers.positionIncrementGap, meta); } - private NamedAnalyzer wrapAnalyzer(NamedAnalyzer in, int positionIncrementGap) { - return new NamedAnalyzer(in.name(), AnalyzerScope.INDEX, - new AnnotationAnalyzerWrapper(in.analyzer()), positionIncrementGap); - } - private AnnotatedTextFieldType buildFieldType(FieldType fieldType, BuilderContext context) { - int posGap; - if (positionIncrementGap.get() == POSITION_INCREMENT_GAP_USE_ANALYZER) { - posGap = TextFieldMapper.Defaults.POSITION_INCREMENT_GAP; - } else { - if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { - throw new IllegalArgumentException("Cannot set position_increment_gap on field [" + name() - + "] without positions enabled"); - } - posGap = positionIncrementGap.get(); - } TextSearchInfo tsi = new TextSearchInfo( fieldType, similarity.get(), - wrapAnalyzer(analyzers.getSearchAnalyzer(), posGap), - wrapAnalyzer(analyzers.getSearchQuoteAnalyzer(), posGap)); + wrapAnalyzer(analyzers.getSearchAnalyzer()), + wrapAnalyzer(analyzers.getSearchQuoteAnalyzer())); AnnotatedTextFieldType ft = new AnnotatedTextFieldType( buildFullName(context), store.getValue(), tsi, meta.getValue()); - ft.setIndexAnalyzer(wrapAnalyzer(analyzers.getIndexAnalyzer(), posGap)); return ft; } @@ -148,6 +128,12 @@ public AnnotatedTextFieldMapper build(BuilderContext context) { if (fieldType.indexOptions() == IndexOptions.NONE ) { throw new IllegalArgumentException("[" + CONTENT_TYPE + "] fields must be indexed"); } + if (analyzers.positionIncrementGap.get() != TextParams.POSITION_INCREMENT_GAP_USE_ANALYZER) { + if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { + throw new IllegalArgumentException("Cannot set position_increment_gap on field [" + + name + "] without positions enabled"); + } + } return new AnnotatedTextFieldMapper( name, fieldType, buildFieldType(fieldType, context), multiFieldsBuilder.build(this, context), copyTo.build(), this); @@ -524,7 +510,7 @@ public String typeName() { protected AnnotatedTextFieldMapper(String simpleName, FieldType fieldType, AnnotatedTextFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo, Builder builder) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, wrapAnalyzer(builder.analyzers.getIndexAnalyzer()), multiFields, copyTo); assert fieldType.tokenized(); this.fieldType = fieldType; this.builder = builder; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java index ce772d9c4f789..4340d04fc90b4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java @@ -180,7 +180,7 @@ private static Analyzer getAnalyzer(AnalyzeAction.Request request, AnalysisRegis MappedFieldType fieldType = indexService.mapperService().fieldType(request.field()); if (fieldType != null) { if (fieldType instanceof StringFieldType) { - return fieldType.indexAnalyzer(); + return indexService.mapperService().indexAnalyzer(); } else { throw new IllegalArgumentException("Can't process field [" + request.field() + "], Analysis requests are only supported on tokenized fields"); @@ -220,10 +220,8 @@ private static List simpleAnalyze(AnalyzeAction.Requ List tokens = new ArrayList<>(); int lastPosition = -1; int lastOffset = 0; - // Note that we always pass "" as the field to the various Analyzer methods, because - // the analyzers we use here are all field-specific and so ignore this parameter for (String text : request.text()) { - try (TokenStream stream = analyzer.tokenStream("", text)) { + try (TokenStream stream = analyzer.tokenStream(request.field(), text)) { stream.reset(); CharTermAttribute term = stream.addAttribute(CharTermAttribute.class); PositionIncrementAttribute posIncr = stream.addAttribute(PositionIncrementAttribute.class); @@ -244,8 +242,8 @@ private static List simpleAnalyze(AnalyzeAction.Requ lastOffset += offset.endOffset(); lastPosition += posIncr.getPositionIncrement(); - lastPosition += analyzer.getPositionIncrementGap(""); - lastOffset += analyzer.getOffsetGap(""); + lastPosition += analyzer.getPositionIncrementGap(request.field()); + lastOffset += analyzer.getOffsetGap(request.field()); } catch (IOException e) { throw new ElasticsearchException("failed to analyze", e); } @@ -345,13 +343,16 @@ private static AnalyzeAction.DetailAnalyzeResponse detailAnalyze(AnalyzeAction.R if (analyzer instanceof NamedAnalyzer) { name = ((NamedAnalyzer) analyzer).name(); } else { - name = analyzer.getClass().getName(); + name = request.field(); } TokenListCreator tokenListCreator = new TokenListCreator(maxTokenCount); for (String text : request.text()) { - tokenListCreator.analyze(analyzer.tokenStream("", text), includeAttributes, analyzer.getPositionIncrementGap(""), - analyzer.getOffsetGap("")); + tokenListCreator.analyze( + analyzer.tokenStream(request.field(), text), + includeAttributes, + analyzer.getPositionIncrementGap(request.field()), + analyzer.getOffsetGap(request.field())); } detailResponse = new AnalyzeAction.DetailAnalyzeResponse(new AnalyzeAction.AnalyzeTokenList(name, tokenListCreator.getArrayTokens())); diff --git a/server/src/main/java/org/elasticsearch/index/analysis/FieldNameAnalyzer.java b/server/src/main/java/org/elasticsearch/index/analysis/FieldNameAnalyzer.java index dbb355ea51ca5..955d213dbaa55 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/FieldNameAnalyzer.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/FieldNameAnalyzer.java @@ -48,4 +48,20 @@ protected Analyzer getWrappedAnalyzer(String fieldName) { // Fields need to be explicitly added throw new IllegalArgumentException("Field [" + fieldName + "] has no associated analyzer"); } + + public boolean containsBrokenAnalysis(String field) { + Analyzer analyzer = getWrappedAnalyzer(field); + if (analyzer instanceof NamedAnalyzer) { + analyzer = ((NamedAnalyzer) analyzer).analyzer(); + } + if (analyzer instanceof AnalyzerComponentsProvider) { + final TokenFilterFactory[] tokenFilters = ((AnalyzerComponentsProvider) analyzer).getComponents().getTokenFilters(); + for (TokenFilterFactory tokenFilterFactory : tokenFilters) { + if (tokenFilterFactory.breaksFastVectorHighlighter()) { + return true; + } + } + } + return false; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index ef70e836fd773..4d066cfe405a3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.support.MapXContentParser; +import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.lookup.SearchLookup; @@ -151,16 +152,24 @@ protected Object parseSourceValue(Object value) { private final Parser parser; protected AbstractGeometryFieldMapper(String simpleName, MappedFieldType mappedFieldType, + Map indexAnalyzers, Explicit ignoreMalformed, Explicit ignoreZValue, MultiFields multiFields, CopyTo copyTo, Indexer indexer, Parser parser) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, indexAnalyzers, multiFields, copyTo); this.ignoreMalformed = ignoreMalformed; this.ignoreZValue = ignoreZValue; this.indexer = indexer; this.parser = parser; } + protected AbstractGeometryFieldMapper(String simpleName, MappedFieldType mappedFieldType, + Explicit ignoreMalformed, Explicit ignoreZValue, + MultiFields multiFields, CopyTo copyTo, + Indexer indexer, Parser parser) { + this(simpleName, mappedFieldType, Collections.emptyMap(), ignoreMalformed, ignoreZValue, multiFields, copyTo, indexer, parser); + } + @Override public AbstractGeometryFieldType fieldType() { return (AbstractGeometryFieldType) mappedFieldType; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java index fad7e7e7ce824..fd127919d8966 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java @@ -21,7 +21,9 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.geo.builders.ShapeBuilder.Orientation; +import org.elasticsearch.index.analysis.NamedAnalyzer; +import java.util.Collections; import java.util.Map; import java.util.function.Function; @@ -61,15 +63,25 @@ protected AbstractShapeGeometryFieldType(String name, boolean isSearchable, bool protected Explicit orientation; protected AbstractShapeGeometryFieldMapper(String simpleName, MappedFieldType mappedFieldType, + Map indexAnalyzers, Explicit ignoreMalformed, Explicit coerce, Explicit ignoreZValue, Explicit orientation, MultiFields multiFields, CopyTo copyTo, Indexer indexer, Parser parser) { - super(simpleName, mappedFieldType, ignoreMalformed, ignoreZValue, multiFields, copyTo, indexer, parser); + super(simpleName, mappedFieldType, indexAnalyzers, ignoreMalformed, ignoreZValue, multiFields, copyTo, indexer, parser); this.coerce = coerce; this.orientation = orientation; } + protected AbstractShapeGeometryFieldMapper(String simpleName, MappedFieldType mappedFieldType, + Explicit ignoreMalformed, Explicit coerce, + Explicit ignoreZValue, Explicit orientation, + MultiFields multiFields, CopyTo copyTo, + Indexer indexer, Parser parser) { + this(simpleName, mappedFieldType, Collections.emptyMap(), + ignoreMalformed, coerce, ignoreZValue, orientation, multiFields, copyTo, indexer, parser); + } + @Override public final boolean parsesArrayValue() { return false; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index bf10f9f16dcee..17f9463fec270 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -29,6 +29,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -220,7 +221,7 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower protected BooleanFieldMapper(String simpleName, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo, Builder builder) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, multiFields, copyTo); this.nullValue = builder.nullValue.getValue(); this.stored = builder.stored.getValue(); this.indexed = builder.indexed.getValue(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index 1c007fb6f4065..3386fe988ae8a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -87,7 +87,7 @@ public class CompletionFieldMapper extends FieldMapper { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), defaultAnalyzer, indexVersionCreated).init(this); + return new Builder(simpleName(), builder.defaultAnalyzer, builder.indexVersionCreated).init(this); } public static class Defaults { @@ -112,8 +112,8 @@ public static class Fields { public static final String CONTENT_FIELD_NAME_CONTEXTS = "contexts"; } - private static CompletionFieldMapper toType(FieldMapper in) { - return (CompletionFieldMapper) in; + private static Builder builder(FieldMapper in) { + return ((CompletionFieldMapper)in).builder; } /** @@ -124,13 +124,13 @@ public static class Builder extends FieldMapper.Builder { private final Parameter analyzer; private final Parameter searchAnalyzer; private final Parameter preserveSeparators = Parameter.boolParam("preserve_separators", false, - m -> toType(m).preserveSeparators, Defaults.DEFAULT_PRESERVE_SEPARATORS) + m -> builder(m).preserveSeparators.get(), Defaults.DEFAULT_PRESERVE_SEPARATORS) .alwaysSerialize(); private final Parameter preservePosInc = Parameter.boolParam("preserve_position_increments", false, - m -> toType(m).preservePosInc, Defaults.DEFAULT_POSITION_INCREMENTS) + m -> builder(m).preservePosInc.get(), Defaults.DEFAULT_POSITION_INCREMENTS) .alwaysSerialize(); private final Parameter contexts = new Parameter<>("contexts", false, () -> null, - (n, c, o) -> ContextMappings.load(o, c.indexVersionCreated()), m -> toType(m).contexts) + (n, c, o) -> ContextMappings.load(o, c.indexVersionCreated()), m -> builder(m).contexts.get()) .setSerializer((b, n, c) -> { if (c == null) { return; @@ -140,7 +140,7 @@ public static class Builder extends FieldMapper.Builder { b.endArray(); }, Objects::toString); private final Parameter maxInputLength = Parameter.intParam("max_input_length", true, - m -> toType(m).maxInputLength, Defaults.DEFAULT_MAX_INPUT_LENGTH) + m -> builder(m).maxInputLength.get(), Defaults.DEFAULT_MAX_INPUT_LENGTH) .addDeprecatedName("max_input_len") .setValidator(Builder::validateInputLength) .alwaysSerialize(); @@ -158,10 +158,10 @@ public Builder(String name, NamedAnalyzer defaultAnalyzer, Version indexVersionC super(name); this.defaultAnalyzer = defaultAnalyzer; this.indexVersionCreated = indexVersionCreated; - this.analyzer = Parameter.analyzerParam("analyzer", false, m -> toType(m).analyzer, () -> defaultAnalyzer) + this.analyzer = Parameter.analyzerParam("analyzer", false, m -> builder(m).analyzer.get(), () -> defaultAnalyzer) .alwaysSerialize(); this.searchAnalyzer - = Parameter.analyzerParam("search_analyzer", true, m -> toType(m).searchAnalyzer, analyzer::getValue); + = Parameter.analyzerParam("search_analyzer", true, m -> builder(m).searchAnalyzer.get(), analyzer::getValue); } private static void validateInputLength(int maxInputLength) { @@ -175,6 +175,11 @@ protected List> getParameters() { return List.of(analyzer, searchAnalyzer, preserveSeparators, preservePosInc, maxInputLength, contexts, meta); } + NamedAnalyzer buildAnalyzer() { + return new NamedAnalyzer(analyzer.get().name(), AnalyzerScope.INDEX, + new CompletionAnalyzer(analyzer.get(), preserveSeparators.get(), preservePosInc.get())); + } + @Override public CompletionFieldMapper build(BuilderContext context) { checkCompletionContextsLimit(context); @@ -184,11 +189,8 @@ public CompletionFieldMapper build(BuilderContext context) { CompletionFieldType ft = new CompletionFieldType(buildFullName(context), completionAnalyzer, meta.getValue()); ft.setContextMappings(contexts.getValue()); - ft.setPreservePositionIncrements(preservePosInc.getValue()); - ft.setPreserveSep(preserveSeparators.getValue()); - ft.setIndexAnalyzer(analyzer.getValue()); - return new CompletionFieldMapper(name, ft, defaultAnalyzer, - multiFieldsBuilder.build(this, context), copyTo.build(), indexVersionCreated, this); + return new CompletionFieldMapper(name, ft, + multiFieldsBuilder.build(this, context), copyTo.build(), this); } private void checkCompletionContextsLimit(BuilderContext context) { @@ -218,37 +220,16 @@ public static final class CompletionFieldType extends TermBasedFieldType { private static PostingsFormat postingsFormat; - private boolean preserveSep = Defaults.DEFAULT_PRESERVE_SEPARATORS; - private boolean preservePositionIncrements = Defaults.DEFAULT_POSITION_INCREMENTS; private ContextMappings contextMappings = null; public CompletionFieldType(String name, NamedAnalyzer searchAnalyzer, Map meta) { super(name, true, false, false, new TextSearchInfo(Defaults.FIELD_TYPE, null, searchAnalyzer, searchAnalyzer), meta); } - public void setPreserveSep(boolean preserveSep) { - this.preserveSep = preserveSep; - } - - public void setPreservePositionIncrements(boolean preservePositionIncrements) { - this.preservePositionIncrements = preservePositionIncrements; - } - public void setContextMappings(ContextMappings contextMappings) { this.contextMappings = contextMappings; } - @Override - public NamedAnalyzer indexAnalyzer() { - final NamedAnalyzer indexAnalyzer = super.indexAnalyzer(); - if (indexAnalyzer != null && !(indexAnalyzer.analyzer() instanceof CompletionAnalyzer)) { - return new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX, - new CompletionAnalyzer(indexAnalyzer, preserveSep, preservePositionIncrements)); - - } - return indexAnalyzer; - } - /** * @return true if there are one or more context mappings defined * for this field type @@ -327,25 +308,13 @@ protected List parseSourceValue(Object value) { } private final int maxInputLength; - private final boolean preserveSeparators; - private final boolean preservePosInc; - private final NamedAnalyzer defaultAnalyzer; - private final NamedAnalyzer analyzer; - private final NamedAnalyzer searchAnalyzer; - private final ContextMappings contexts; - private final Version indexVersionCreated; - - public CompletionFieldMapper(String simpleName, MappedFieldType mappedFieldType, NamedAnalyzer defaultAnalyzer, - MultiFields multiFields, CopyTo copyTo, Version indexVersionCreated, Builder builder) { - super(simpleName, mappedFieldType, multiFields, copyTo); - this.defaultAnalyzer = defaultAnalyzer; + private final Builder builder; + + public CompletionFieldMapper(String simpleName, MappedFieldType mappedFieldType, + MultiFields multiFields, CopyTo copyTo, Builder builder) { + super(simpleName, mappedFieldType, builder.buildAnalyzer(), multiFields, copyTo); + this.builder = builder; this.maxInputLength = builder.maxInputLength.getValue(); - this.preserveSeparators = builder.preserveSeparators.getValue(); - this.preservePosInc = builder.preservePosInc.getValue(); - this.analyzer = builder.analyzer.getValue(); - this.searchAnalyzer = builder.searchAnalyzer.getValue(); - this.contexts = builder.contexts.getValue(); - this.indexVersionCreated = indexVersionCreated; } @Override @@ -359,7 +328,7 @@ public boolean parsesArrayValue() { } int getMaxInputLength() { - return maxInputLength; + return builder.maxInputLength.get(); } /** @@ -403,7 +372,7 @@ public void parse(ParseContext context) throws IOException { } // truncate input if (input.length() > maxInputLength) { - int len = Math.min(maxInputLength, input.length()); + int len = maxInputLength; if (Character.isHighSurrogate(input.charAt(len - 1))) { assert input.length() >= len + 1 && Character.isLowSurrogate(input.charAt(len)); len += 1; @@ -509,7 +478,7 @@ private void parse(ParseContext parseContext, Token token, ContextMappings contextMappings = fieldType().getContextMappings(); XContentParser.Token currentToken = parser.currentToken(); if (currentToken == XContentParser.Token.START_OBJECT) { - ContextMapping contextMapping = null; + ContextMapping contextMapping = null; String fieldName = null; while ((currentToken = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (currentToken == XContentParser.Token.FIELD_NAME) { @@ -569,7 +538,7 @@ protected String contentType() { public void doValidate(MappingLookup mappers) { if (fieldType().hasContextMappings()) { for (ContextMapping contextMapping : fieldType().getContextMappings()) { - contextMapping.validateReferences(indexVersionCreated, s -> mappers.fieldTypes().get(s)); + contextMapping.validateReferences(builder.indexVersionCreated, s -> mappers.fieldTypes().get(s)); } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 87f0d1b4d3244..3bcc5d651a07b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -120,7 +120,7 @@ private DocumentMapper(IndexSettings indexSettings, this.documentParser = documentParser; this.indexSettings = indexSettings; this.indexAnalyzers = indexAnalyzers; - this.fieldMappers = MappingLookup.fromMapping(this.mapping, indexAnalyzers.getDefaultIndexAnalyzer()); + this.fieldMappers = MappingLookup.fromMapping(this.mapping); try { mappingSource = new CompressedXContent(this, XContentType.JSON, ToXContent.EMPTY_PARAMS); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicKeyFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicKeyFieldMapper.java index 2811d366a13b1..2463cc0d48211 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicKeyFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicKeyFieldMapper.java @@ -19,6 +19,8 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.index.analysis.NamedAnalyzer; + /** * A field mapper that supports lookup of dynamic sub-keys. If the field mapper is named 'my_field', * then a user is able to search on the field in both of the following ways: @@ -41,8 +43,9 @@ public abstract class DynamicKeyFieldMapper extends FieldMapper { public DynamicKeyFieldMapper(String simpleName, MappedFieldType defaultFieldType, + NamedAnalyzer indexAnalyzer, CopyTo copyTo) { - super(simpleName, defaultFieldType, MultiFields.empty(), copyTo); + super(simpleName, defaultFieldType, indexAnalyzer, MultiFields.empty(), copyTo); } public abstract MappedFieldType keyedFieldType(String key); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 69ec5ba5874a6..36f45268880b8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -64,17 +64,55 @@ public abstract class FieldMapper extends Mapper implements Cloneable { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(FieldMapper.class); - protected MappedFieldType mappedFieldType; - protected MultiFields multiFields; - protected CopyTo copyTo; + protected final MappedFieldType mappedFieldType; + protected final Map indexAnalyzers; + protected final MultiFields multiFields; + protected final CopyTo copyTo; + /** + * Create a FieldMapper with no index analyzers + * @param simpleName the leaf name of the mapper + * @param mappedFieldType the MappedFieldType associated with this mapper + * @param multiFields sub fields of this mapper + * @param copyTo copyTo fields of this mapper + */ + protected FieldMapper(String simpleName, MappedFieldType mappedFieldType, + MultiFields multiFields, CopyTo copyTo) { + this(simpleName, mappedFieldType, Collections.emptyMap(), multiFields, copyTo); + } + + /** + * Create a FieldMapper with a single associated index analyzer + * @param simpleName the leaf name of the mapper + * @param mappedFieldType the MappedFieldType associated with this mapper + * @param indexAnalyzer the index-time analyzer to use for this field + * @param multiFields sub fields of this mapper + * @param copyTo copyTo fields of this mapper + */ protected FieldMapper(String simpleName, MappedFieldType mappedFieldType, + NamedAnalyzer indexAnalyzer, + MultiFields multiFields, CopyTo copyTo) { + this(simpleName, mappedFieldType, Collections.singletonMap(mappedFieldType.name(), indexAnalyzer), multiFields, copyTo); + } + + /** + * Create a FieldMapper that indexes into multiple analyzed fields + * @param simpleName the leaf name of the mapper + * @param mappedFieldType the MappedFieldType associated with this mapper + * @param indexAnalyzers a map of field names to analyzers, one for each analyzed field + * the mapper will add + * @param multiFields sub fields of this mapper + * @param copyTo copyTo fields of this mapper + */ + protected FieldMapper(String simpleName, MappedFieldType mappedFieldType, + Map indexAnalyzers, MultiFields multiFields, CopyTo copyTo) { super(simpleName); if (mappedFieldType.name().isEmpty()) { throw new IllegalArgumentException("name cannot be empty string"); } this.mappedFieldType = mappedFieldType; + this.indexAnalyzers = indexAnalyzers; this.multiFields = multiFields; this.copyTo = Objects.requireNonNull(copyTo); } @@ -263,18 +301,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder.endObject(); } - protected boolean indexedByDefault() { - return true; - } - - protected boolean docValuesByDefault() { - return true; - } - - protected boolean storedByDefault() { - return false; - } - protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { builder.field("type", contentType()); getMergeBuilder().toXContent(builder, includeDefaults); @@ -284,6 +310,10 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, protected abstract String contentType(); + public final Map indexAnalyzers() { + return indexAnalyzers; + } + public static class MultiFields implements Iterable { public static MultiFields empty() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index 131817b0d4f1c..a181bb0404dcb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -105,7 +105,6 @@ static final class IdFieldType extends TermBasedFieldType { IdFieldType(BooleanSupplier fieldDataEnabled) { super(NAME, true, true, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); this.fieldDataEnabled = fieldDataEnabled; - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); } @Override @@ -156,11 +155,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S + IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey()); } final IndexFieldData.Builder fieldDataBuilder = new PagedBytesIndexFieldData.Builder( - name(), - TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY, - TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY, - TextFieldMapper.Defaults.FIELDDATA_MIN_SEGMENT_SIZE, - CoreValuesSourceType.BYTES); + name(), + TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY, + TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY, + TextFieldMapper.Defaults.FIELDDATA_MIN_SEGMENT_SIZE, + CoreValuesSourceType.BYTES); return new IndexFieldData.Builder() { @Override public IndexFieldData build( @@ -200,7 +199,8 @@ public SortField sortField(Object missingValue, MultiValueMode sortMode, Nested @Override public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, MultiValueMode sortMode, - Nested nested, SortOrder sortOrder, DocValueFormat format, int bucketSize, BucketedSort.ExtraData extra) { + Nested nested, SortOrder sortOrder, DocValueFormat format, + int bucketSize, BucketedSort.ExtraData extra) { throw new UnsupportedOperationException("can't sort on the [" + CONTENT_TYPE + "] field"); } }; @@ -236,7 +236,7 @@ public SortedBinaryDocValues getBytesValues() { public BytesRef nextValue() throws IOException { BytesRef encoded = inValues.nextValue(); return new BytesRef(Uid.decodeId( - Arrays.copyOfRange(encoded.bytes, encoded.offset, encoded.offset + encoded.length))); + Arrays.copyOfRange(encoded.bytes, encoded.offset, encoded.offset + encoded.length))); } @Override @@ -258,7 +258,7 @@ public boolean advanceExact(int doc) throws IOException { } private IdFieldMapper(BooleanSupplier fieldDataEnabled) { - super(new IdFieldType(fieldDataEnabled)); + super(new IdFieldType(fieldDataEnabled), Lucene.KEYWORD_ANALYZER); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 1b9918fbd9cf3..22713d905f83b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -438,4 +438,5 @@ protected void parseCreateField(ParseContext context) throws IOException { public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), ignoreMalformedByDefault, indexCreatedVersion).init(this); } + } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 293360b7cb81e..5822eb625217f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -179,6 +179,7 @@ public static final class KeywordFieldType extends StringFieldType { private final int ignoreAbove; private final String nullValue; + private final NamedAnalyzer normalizer; public KeywordFieldType(String name, FieldType fieldType, NamedAnalyzer normalizer, NamedAnalyzer searchAnalyzer, Builder builder) { @@ -189,14 +190,14 @@ public KeywordFieldType(String name, FieldType fieldType, new TextSearchInfo(fieldType, builder.similarity.getValue(), searchAnalyzer, searchAnalyzer), builder.meta.getValue()); setEagerGlobalOrdinals(builder.eagerGlobalOrdinals.getValue()); - setIndexAnalyzer(normalizer); + this.normalizer = normalizer; this.ignoreAbove = builder.ignoreAbove.getValue(); this.nullValue = builder.nullValue.getValue(); } public KeywordFieldType(String name, boolean isSearchable, boolean hasDocValues, Map meta) { super(name, isSearchable, false, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); + this.normalizer = Lucene.KEYWORD_ANALYZER; this.ignoreAbove = Integer.MAX_VALUE; this.nullValue = null; } @@ -210,12 +211,14 @@ public KeywordFieldType(String name, FieldType fieldType) { false, false, new TextSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), Collections.emptyMap()); + this.normalizer = Lucene.KEYWORD_ANALYZER; this.ignoreAbove = Integer.MAX_VALUE; this.nullValue = null; } public KeywordFieldType(String name, NamedAnalyzer analyzer) { super(name, true, false, true, new TextSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap()); + this.normalizer = Lucene.KEYWORD_ANALYZER; this.ignoreAbove = Integer.MAX_VALUE; this.nullValue = null; } @@ -226,7 +229,7 @@ public String typeName() { } NamedAnalyzer normalizer() { - return indexAnalyzer(); + return normalizer; } @Override @@ -312,9 +315,9 @@ public CollapseType collapseType() { private final IndexAnalyzers indexAnalyzers; - protected KeywordFieldMapper(String simpleName, FieldType fieldType, MappedFieldType mappedFieldType, + protected KeywordFieldMapper(String simpleName, FieldType fieldType, KeywordFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo, Builder builder) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, mappedFieldType.normalizer, multiFields, copyTo); assert fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) <= 0; this.indexed = builder.indexed.getValue(); this.hasDocValues = builder.hasDocValues.getValue(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java index 9e1a935b4ab89..d4b4b71580f73 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java @@ -38,6 +38,7 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.geo.builders.ShapeBuilder.Orientation; import org.elasticsearch.common.geo.parsers.ShapeParser; +import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.support.XContentMapValues; @@ -459,7 +460,7 @@ public LegacyGeoShapeFieldMapper(String simpleName, MappedFieldType mappedFieldT MultiFields multiFields, CopyTo copyTo, LegacyGeoShapeIndexer indexer, LegacyGeoShapeParser parser, Builder builder) { - super(simpleName, mappedFieldType, + super(simpleName, mappedFieldType, Collections.singletonMap(mappedFieldType.name(), Lucene.KEYWORD_ANALYZER), builder.ignoreMalformed.get(), builder.coerce.get(), builder.ignoreZValue.get(), builder.orientation.get(), multiFields, copyTo, indexer, parser); this.indexCreatedVersion = builder.indexCreatedVersion; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index db80fcae21dd1..6cf79d4abae74 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -123,14 +123,6 @@ public boolean hasDocValues() { return docValues; } - public NamedAnalyzer indexAnalyzer() { - return indexAnalyzer; - } - - public void setIndexAnalyzer(NamedAnalyzer analyzer) { - this.indexAnalyzer = analyzer; - } - /** * Returns the collapse type of the field * CollapseType.NONE means the field can'be used for collapsing. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index c3e80c01a70f8..766955f0e32a1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -121,7 +121,7 @@ public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers, super(indexSettings); this.indexVersionCreated = indexSettings.getIndexVersionCreated(); this.indexAnalyzers = indexAnalyzers; - this.indexAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultIndexAnalyzer(), MappedFieldType::indexAnalyzer); + this.indexAnalyzer = new MapperAnalyzerWrapper(); this.mapperRegistry = mapperRegistry; Function parserContextFunction = dateFormatter -> new Mapper.TypeParser.ParserContext(similarityService::getSimilarity, mapperRegistry.getMapperParsers()::get, @@ -444,6 +444,10 @@ public Analyzer indexAnalyzer() { return this.indexAnalyzer; } + public boolean containsBrokenAnalysis(String field) { + return this.indexAnalyzer.containsBrokenAnalysis(field); + } + @Override public void close() throws IOException { indexAnalyzers.close(); @@ -476,25 +480,17 @@ public boolean isMetadataField(String field) { /** An analyzer wrapper that can lookup fields within the index mappings */ final class MapperAnalyzerWrapper extends DelegatingAnalyzerWrapper { - private final Analyzer defaultAnalyzer; - private final Function extractAnalyzer; - - MapperAnalyzerWrapper(Analyzer defaultAnalyzer, Function extractAnalyzer) { + MapperAnalyzerWrapper() { super(Analyzer.PER_FIELD_REUSE_STRATEGY); - this.defaultAnalyzer = defaultAnalyzer; - this.extractAnalyzer = extractAnalyzer; } @Override protected Analyzer getWrappedAnalyzer(String fieldName) { - MappedFieldType fieldType = fieldType(fieldName); - if (fieldType != null) { - Analyzer analyzer = extractAnalyzer.apply(fieldType); - if (analyzer != null) { - return analyzer; - } - } - return defaultAnalyzer; + return mapper.mappers().indexAnalyzer(); + } + + boolean containsBrokenAnalysis(String field) { + return mapper.mappers().indexAnalyzer().containsBrokenAnalysis(field); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java index 9bff34bbd88c6..c9b0ce0d3d7a2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -42,14 +42,7 @@ public final class MappingLookup implements Iterable { private final int metadataFieldCount; private final FieldNameAnalyzer indexAnalyzer; - private static void put(Map analyzers, String key, Analyzer value, Analyzer defaultValue) { - if (value == null) { - value = defaultValue; - } - analyzers.put(key, value); - } - - public static MappingLookup fromMapping(Mapping mapping, Analyzer defaultIndex) { + public static MappingLookup fromMapping(Mapping mapping) { List newObjectMappers = new ArrayList<>(); List newFieldMappers = new ArrayList<>(); List newFieldAliasMappers = new ArrayList<>(); @@ -59,7 +52,7 @@ public static MappingLookup fromMapping(Mapping mapping, Analyzer defaultIndex) } } collect(mapping.root, newObjectMappers, newFieldMappers, newFieldAliasMappers); - return new MappingLookup(newFieldMappers, newObjectMappers, newFieldAliasMappers, mapping.metadataMappers.length, defaultIndex); + return new MappingLookup(newFieldMappers, newObjectMappers, newFieldAliasMappers, mapping.metadataMappers.length); } private static void collect(Mapper mapper, Collection objectMappers, @@ -86,8 +79,7 @@ private static void collect(Mapper mapper, Collection objectMapper public MappingLookup(Collection mappers, Collection objectMappers, Collection aliasMappers, - int metadataFieldCount, - Analyzer defaultIndex) { + int metadataFieldCount) { Map fieldMappers = new HashMap<>(); Map indexAnalyzers = new HashMap<>(); Map objects = new HashMap<>(); @@ -110,8 +102,7 @@ public MappingLookup(Collection mappers, if (fieldMappers.put(mapper.name(), mapper) != null) { throw new MapperParsingException("Field [" + mapper.name() + "] is defined more than once"); } - MappedFieldType fieldType = mapper.fieldType(); - put(indexAnalyzers, fieldType.name(), fieldType.indexAnalyzer(), defaultIndex); + indexAnalyzers.putAll(mapper.indexAnalyzers()); } this.metadataFieldCount = metadataFieldCount; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java index 5f70a970aaace..fc76756206188 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.analysis.NamedAnalyzer; import java.io.IOException; import java.util.Map; @@ -135,6 +136,10 @@ protected MetadataFieldMapper(MappedFieldType mappedFieldType) { super(mappedFieldType.name(), mappedFieldType, MultiFields.empty(), CopyTo.empty()); } + protected MetadataFieldMapper(MappedFieldType mappedFieldType, NamedAnalyzer indexAnalyzer) { + super(mappedFieldType.name(), mappedFieldType, indexAnalyzer, MultiFields.empty(), CopyTo.empty()); + } + @Override public FieldMapper.Builder getMergeBuilder() { return null; // by default, things can't be configured so we have no builder diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index f180caad813cb..94953c2052218 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -37,7 +37,6 @@ import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Numbers; -import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; @@ -898,7 +897,6 @@ public NumberFieldType(String name, NumberType type, boolean isSearchable, boole this.type = Objects.requireNonNull(type); this.coerce = coerce; this.nullValue = nullValue; - this.setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); // allows number fields in significant text aggs - do we need this? } NumberFieldType(String name, Builder builder) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index dc685d9573778..db1d0bc7be2cb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.geo.ShapeRelation; -import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; @@ -158,7 +157,6 @@ public RangeFieldType(String name, RangeType type, boolean indexed, boolean stor this.rangeType = Objects.requireNonNull(type); dateTimeFormatter = null; dateMathParser = null; - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); this.coerce = coerce; } @@ -172,7 +170,6 @@ public RangeFieldType(String name, boolean indexed, boolean stored, boolean has this.rangeType = RangeType.DATE; this.dateTimeFormatter = Objects.requireNonNull(formatter); this.dateMathParser = dateTimeFormatter.toDateMathParser(); - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); this.coerce = coerce; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java index cbf894a438df4..3516239134285 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java @@ -87,7 +87,6 @@ static final class RoutingFieldType extends StringFieldType { private RoutingFieldType() { super(NAME, true, true, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); } @Override @@ -104,7 +103,7 @@ public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup lookup, private final boolean required; private RoutingFieldMapper(boolean required) { - super(RoutingFieldType.INSTANCE); + super(RoutingFieldType.INSTANCE, Lucene.KEYWORD_ANALYZER); this.required = required; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 07fb1341d28e9..c52062d8e4d08 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -79,6 +79,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -90,7 +91,6 @@ public class TextFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "text"; - private static final int POSITION_INCREMENT_GAP_USE_ANALYZER = -1; private static final String FAST_PHRASE_SUFFIX = "._index_phrase"; public static class Defaults { @@ -251,9 +251,6 @@ public static class Builder extends FieldMapper.Builder { final Parameter norms = TextParams.norms(true, m -> builder(m).norms.getValue()); final Parameter termVectors = TextParams.termVectors(m -> builder(m).termVectors.getValue()); - final Parameter positionIncrementGap = Parameter.intParam("position_increment_gap", false, - m -> builder(m).positionIncrementGap.getValue(), POSITION_INCREMENT_GAP_USE_ANALYZER); - final Parameter fieldData = Parameter.boolParam("fielddata", true, m -> builder(m).fieldData.getValue(), false); final Parameter freqFilter = new Parameter<>("fielddata_frequency_filter", true, @@ -277,7 +274,7 @@ public Builder(String name, Supplier defaultAnalyzer) { public Builder(String name, Version indexCreatedVersion, Supplier defaultAnalyzer) { super(name); this.indexCreatedVersion = indexCreatedVersion; - this.analyzers = new TextParams.Analyzers(defaultAnalyzer); + this.analyzers = new TextParams.Analyzers(defaultAnalyzer, m -> builder(m).analyzers); } public Builder index(boolean index) { @@ -309,28 +306,23 @@ public Builder addMultiField(Mapper.Builder builder) { protected List> getParameters() { return Arrays.asList(index, store, indexOptions, norms, termVectors, analyzers.indexAnalyzer, analyzers.searchAnalyzer, analyzers.searchQuoteAnalyzer, similarity, - positionIncrementGap, + analyzers.positionIncrementGap, fieldData, freqFilter, eagerGlobalOrdinals, indexPhrases, indexPrefixes, meta); } private TextFieldType buildFieldType(FieldType fieldType, BuilderContext context) { - NamedAnalyzer indexAnalyzer = analyzers.getIndexAnalyzer(); NamedAnalyzer searchAnalyzer = analyzers.getSearchAnalyzer(); NamedAnalyzer searchQuoteAnalyzer = analyzers.getSearchQuoteAnalyzer(); - if (positionIncrementGap.get() != POSITION_INCREMENT_GAP_USE_ANALYZER) { + if (analyzers.positionIncrementGap.get() != TextParams.POSITION_INCREMENT_GAP_USE_ANALYZER) { if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { throw new IllegalArgumentException("Cannot set position_increment_gap on field [" + name + "] without positions enabled"); } - indexAnalyzer = new NamedAnalyzer(indexAnalyzer, positionIncrementGap.get()); - searchAnalyzer = new NamedAnalyzer(searchAnalyzer, positionIncrementGap.get()); - searchQuoteAnalyzer = new NamedAnalyzer(searchQuoteAnalyzer, positionIncrementGap.get()); } TextSearchInfo tsi = new TextSearchInfo(fieldType, similarity.getValue(), searchAnalyzer, searchQuoteAnalyzer); TextFieldType ft = new TextFieldType(buildFullName(context), index.getValue(), store.getValue(), tsi, meta.getValue()); - ft.setIndexAnalyzer(indexAnalyzer); ft.setEagerGlobalOrdinals(eagerGlobalOrdinals.getValue()); if (fieldData.getValue()) { ft.setFielddata(true, freqFilter.getValue()); @@ -367,9 +359,12 @@ private PrefixFieldMapper buildPrefixMapper(BuilderContext context, FieldType fi pft.setStoreTermVectorOffsets(true); } PrefixFieldType prefixFieldType = new PrefixFieldType(tft, fullName + "._index_prefix", indexPrefixes.get()); - prefixFieldType.setAnalyzer(analyzers.getIndexAnalyzer()); tft.setPrefixFieldType(prefixFieldType); - return new PrefixFieldMapper(pft, prefixFieldType); + return new PrefixFieldMapper(pft, prefixFieldType, new PrefixWrappedAnalyzer( + analyzers.getIndexAnalyzer().analyzer(), + analyzers.positionIncrementGap.get(), + prefixFieldType.minChars, + prefixFieldType.maxChars)); } private PhraseFieldMapper buildPhraseMapper(FieldType fieldType, TextFieldType parent) { @@ -384,15 +379,39 @@ private PhraseFieldMapper buildPhraseMapper(FieldType fieldType, TextFieldType p } FieldType phraseFieldType = new FieldType(fieldType); parent.setIndexPhrases(); - return new PhraseFieldMapper(phraseFieldType, new PhraseFieldType(parent)); + PhraseWrappedAnalyzer a + = new PhraseWrappedAnalyzer(analyzers.getIndexAnalyzer().analyzer(), analyzers.positionIncrementGap.get()); + return new PhraseFieldMapper(phraseFieldType, new PhraseFieldType(parent), a); + } + + public Map indexAnalyzers(String name, + PhraseFieldMapper phraseFieldMapper, + PrefixFieldMapper prefixFieldMapper) { + Map analyzers = new HashMap<>(); + NamedAnalyzer main = this.analyzers.getIndexAnalyzer(); + analyzers.put(name, main); + if (phraseFieldMapper != null) { + analyzers.put( + phraseFieldMapper.name(), + new NamedAnalyzer(main.name() + "_phrase", AnalyzerScope.INDEX, phraseFieldMapper.analyzer)); + } + if (prefixFieldMapper != null) { + analyzers.put( + prefixFieldMapper.name(), + new NamedAnalyzer(main.name() + "_prefix", AnalyzerScope.INDEX, prefixFieldMapper.analyzer)); + } + return analyzers; } @Override public TextFieldMapper build(BuilderContext context) { FieldType fieldType = TextParams.buildFieldType(index, store, indexOptions, norms, termVectors); TextFieldType tft = buildFieldType(fieldType, context); + PhraseFieldMapper phraseFieldMapper = buildPhraseMapper(fieldType, tft); + PrefixFieldMapper prefixFieldMapper = buildPrefixMapper(context, fieldType, tft); return new TextFieldMapper(name, fieldType, tft, - buildPrefixMapper(context, fieldType, tft), buildPhraseMapper(fieldType, tft), + indexAnalyzers(tft.name(), phraseFieldMapper, prefixFieldMapper), + prefixFieldMapper, phraseFieldMapper, multiFieldsBuilder.build(this, context), copyTo.build(), this); } } @@ -403,10 +422,17 @@ public TextFieldMapper build(BuilderContext context) { private static class PhraseWrappedAnalyzer extends AnalyzerWrapper { private final Analyzer delegate; + private final int posIncGap; - PhraseWrappedAnalyzer(Analyzer delegate) { + PhraseWrappedAnalyzer(Analyzer delegate, int posIncGap) { super(delegate.getReuseStrategy()); this.delegate = delegate; + this.posIncGap = posIncGap; + } + + @Override + public int getPositionIncrementGap(String fieldName) { + return posIncGap; } @Override @@ -424,11 +450,13 @@ private static class PrefixWrappedAnalyzer extends AnalyzerWrapper { private final int minChars; private final int maxChars; + private final int posIncGap; private final Analyzer delegate; - PrefixWrappedAnalyzer(Analyzer delegate, int minChars, int maxChars) { + PrefixWrappedAnalyzer(Analyzer delegate, int posIncGap, int minChars, int maxChars) { super(delegate.getReuseStrategy()); this.delegate = delegate; + this.posIncGap = posIncGap; this.minChars = minChars; this.maxChars = maxChars; } @@ -438,6 +466,11 @@ protected Analyzer getWrappedAnalyzer(String fieldName) { return delegate; } + @Override + public int getPositionIncrementGap(String fieldName) { + return posIncGap; + } + @Override protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) { TokenFilter filter = new EdgeNGramTokenFilter(components.getTokenStream(), minChars, maxChars, false); @@ -451,14 +484,9 @@ static final class PhraseFieldType extends StringFieldType { PhraseFieldType(TextFieldType parent) { super(parent.name() + FAST_PHRASE_SUFFIX, true, false, false, parent.getTextSearchInfo(), Collections.emptyMap()); - setAnalyzer(parent.indexAnalyzer().name(), parent.indexAnalyzer().analyzer()); this.parent = parent; } - void setAnalyzer(String name, Analyzer delegate) { - setIndexAnalyzer(new NamedAnalyzer(name, AnalyzerScope.INDEX, new PhraseWrappedAnalyzer(delegate))); - } - @Override public String typeName() { return "phrase"; @@ -501,11 +529,6 @@ public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchL return SourceValueFetcher.toString(name(), context, format); } - void setAnalyzer(NamedAnalyzer delegate) { - setIndexAnalyzer(new NamedAnalyzer(delegate.name(), AnalyzerScope.INDEX, - new PrefixWrappedAnalyzer(delegate.analyzer(), minChars, maxChars))); - } - boolean accept(int length) { return length >= minChars - 1 && length <= maxChars; } @@ -569,11 +592,13 @@ public Query existsQuery(QueryShardContext context) { private static final class PhraseFieldMapper extends FieldMapper { + private final Analyzer analyzer; private final FieldType fieldType; - PhraseFieldMapper(FieldType fieldType, PhraseFieldType mappedFieldType) { + PhraseFieldMapper(FieldType fieldType, PhraseFieldType mappedFieldType, PhraseWrappedAnalyzer analyzer) { super(mappedFieldType.name(), mappedFieldType, MultiFields.empty(), CopyTo.empty()); this.fieldType = fieldType; + this.analyzer = analyzer; } @Override @@ -594,10 +619,12 @@ protected String contentType() { private static final class PrefixFieldMapper extends FieldMapper { + private final Analyzer analyzer; private final FieldType fieldType; - protected PrefixFieldMapper(FieldType fieldType, PrefixFieldType mappedFieldType) { + protected PrefixFieldMapper(FieldType fieldType, PrefixFieldType mappedFieldType, Analyzer analyzer) { super(mappedFieldType.name(), mappedFieldType, MultiFields.empty(), CopyTo.empty()); + this.analyzer = analyzer; this.fieldType = fieldType; } @@ -840,10 +867,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S protected TextFieldMapper(String simpleName, FieldType fieldType, TextFieldType mappedFieldType, + Map indexAnalyzers, PrefixFieldMapper prefixFieldMapper, PhraseFieldMapper phraseFieldMapper, MultiFields multiFields, CopyTo copyTo, Builder builder) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, indexAnalyzers, multiFields, copyTo); assert mappedFieldType.getTextSearchInfo().isTokenized(); assert mappedFieldType.hasDocValues() == false; if (fieldType.indexOptions() == IndexOptions.NONE && fieldType().fielddata()) { @@ -1046,7 +1074,7 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, this.builder.analyzers.searchQuoteAnalyzer.toXContent(builder, includeDefaults); this.builder.similarity.toXContent(builder, includeDefaults); this.builder.eagerGlobalOrdinals.toXContent(builder, includeDefaults); - this.builder.positionIncrementGap.toXContent(builder, includeDefaults); + this.builder.analyzers.positionIncrementGap.toXContent(builder, includeDefaults); this.builder.fieldData.toXContent(builder, includeDefaults); this.builder.freqFilter.toXContent(builder, includeDefaults); this.builder.indexPrefixes.toXContent(builder, includeDefaults); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextParams.java b/server/src/main/java/org/elasticsearch/index/mapper/TextParams.java index bd567e11fac74..448558a90faba 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextParams.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextParams.java @@ -35,16 +35,20 @@ */ public final class TextParams { + public static final int POSITION_INCREMENT_GAP_USE_ANALYZER = -1; + private TextParams() {} public static final class Analyzers { public final Parameter indexAnalyzer; public final Parameter searchAnalyzer; public final Parameter searchQuoteAnalyzer; + public final Parameter positionIncrementGap; - public Analyzers(Supplier defaultAnalyzer) { + public Analyzers(Supplier defaultAnalyzer, + Function analyzerInitFunction) { this.indexAnalyzer = Parameter.analyzerParam("analyzer", false, - m -> m.fieldType().indexAnalyzer(), defaultAnalyzer) + m -> analyzerInitFunction.apply(m).indexAnalyzer.get(), defaultAnalyzer) .setSerializerCheck((id, ic, a) -> id || ic || Objects.equals(a, getSearchAnalyzer()) == false || Objects.equals(a, getSearchQuoteAnalyzer()) == false) .setValidator(a -> a.checkAllowedInMode(AnalysisMode.INDEX_TIME)); @@ -57,18 +61,32 @@ public Analyzers(Supplier defaultAnalyzer) { = Parameter.analyzerParam("search_quote_analyzer", true, m -> m.fieldType().getTextSearchInfo().getSearchQuoteAnalyzer(), searchAnalyzer::getValue) .setValidator(a -> a.checkAllowedInMode(AnalysisMode.SEARCH_TIME)); + this.positionIncrementGap = Parameter.intParam("position_increment_gap", false, + m -> analyzerInitFunction.apply(m).positionIncrementGap.get(), POSITION_INCREMENT_GAP_USE_ANALYZER) + .setValidator(v -> { + if (v != POSITION_INCREMENT_GAP_USE_ANALYZER && v < 0) { + throw new MapperParsingException("[position_increment_gap] must be positive, got [" + v + "]"); + } + }); } public NamedAnalyzer getIndexAnalyzer() { - return indexAnalyzer.getValue(); + return wrapAnalyzer(indexAnalyzer.getValue()); } public NamedAnalyzer getSearchAnalyzer() { - return searchAnalyzer.getValue(); + return wrapAnalyzer(searchAnalyzer.getValue()); } public NamedAnalyzer getSearchQuoteAnalyzer() { - return searchQuoteAnalyzer.getValue(); + return wrapAnalyzer(searchQuoteAnalyzer.getValue()); + } + + private NamedAnalyzer wrapAnalyzer(NamedAnalyzer a) { + if (positionIncrementGap.get() == POSITION_INCREMENT_GAP_USE_ANALYZER) { + return a; + } + return new NamedAnalyzer(a, positionIncrementGap.get()); } } diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index 4a8c5c262d053..4082405d8b96d 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -329,6 +329,14 @@ MappedFieldType failIfFieldMappingNotFound(String name, MappedFieldType fieldMap } } + /** + * Does the index analyzer for this field have token filters that may produce + * backwards offsets in term vectors + */ + public boolean containsBrokenAnalysis(String field) { + return mapperService.containsBrokenAnalysis(field); + } + private SearchLookup lookup = null; /** diff --git a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java index 5a8719451299b..12b836a544c90 100644 --- a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java +++ b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java @@ -218,17 +218,11 @@ private static Fields addGeneratedTermVectors(IndexShard indexShard, Engine.GetR private static Analyzer getAnalyzerAtField(IndexShard indexShard, String field, @Nullable Map perFieldAnalyzer) { MapperService mapperService = indexShard.mapperService(); - Analyzer analyzer; if (perFieldAnalyzer != null && perFieldAnalyzer.containsKey(field)) { - analyzer = mapperService.getIndexAnalyzers().get(perFieldAnalyzer.get(field)); + return mapperService.getIndexAnalyzers().get(perFieldAnalyzer.get(field)); } else { - MappedFieldType fieldType = mapperService.fieldType(field); - analyzer = fieldType.indexAnalyzer(); + return mapperService.indexAnalyzer(); } - if (analyzer == null) { - analyzer = mapperService.getIndexAnalyzers().getDefaultIndexAnalyzer(); - } - return analyzer; } private static Set getFieldsToGenerate(Map perAnalyzerField, Fields fieldsObject) { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java index 369ff241c7cff..e6e71630f8938 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java @@ -131,6 +131,7 @@ protected Aggregator createInternal(SearchContext searchContext, Aggregator pare context.lookup().source(), context.bigArrays(), fieldType, + searchContext.getQueryShardContext().getIndexAnalyzer(), sourceFieldNames, filterDuplicateText ); @@ -157,6 +158,7 @@ private static class SignificantTextCollectorSource implements MapStringTermsAgg private final SourceLookup sourceLookup; private final BigArrays bigArrays; private final MappedFieldType fieldType; + private final Analyzer analyzer; private final String[] sourceFieldNames; private ObjectArray dupSequenceSpotters; @@ -164,12 +166,14 @@ private static class SignificantTextCollectorSource implements MapStringTermsAgg SourceLookup sourceLookup, BigArrays bigArrays, MappedFieldType fieldType, + Analyzer analyzer, String[] sourceFieldNames, boolean filterDuplicateText ) { this.sourceLookup = sourceLookup; this.bigArrays = bigArrays; this.fieldType = fieldType; + this.analyzer = analyzer; this.sourceFieldNames = sourceFieldNames; dupSequenceSpotters = filterDuplicateText ? bigArrays.newObjectArray(1) : null; } @@ -223,7 +227,6 @@ private void collectFromSource(int doc, long owningBucketOrd, DuplicateByteSeque return obj.toString(); }) .iterator(); - Analyzer analyzer = fieldType.indexAnalyzer(); while (itr.hasNext()) { TokenStream ts = analyzer.tokenStream(fieldType.name(), itr.next()); processTokenStream(doc, owningBucketOrd, ts, inDocTerms, spotter); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java index 1071b5fcf00e7..0263166059db5 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java @@ -167,6 +167,14 @@ public SearchHighlightContext highlight() { return searchContext.highlight(); } + /** + * Does the index analyzer for this field have token filters that may produce + * backwards offsets in term vectors + */ + public boolean containsBrokenAnalysis(String field) { + return getQueryShardContext().containsBrokenAnalysis(field); + } + /** * Should the response include scores, even if scores were not calculated in the original query */ diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FastVectorHighlighter.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FastVectorHighlighter.java index bc529c83bdfa6..fdb49d16e2c27 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FastVectorHighlighter.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FastVectorHighlighter.java @@ -72,6 +72,7 @@ public HighlightField highlight(FieldHighlightContext fieldContext) throws IOExc FetchSubPhase.HitContext hitContext = fieldContext.hitContext; MappedFieldType fieldType = fieldContext.fieldType; boolean forceSource = fieldContext.forceSource; + boolean fixBrokenAnalysis = fieldContext.context.containsBrokenAnalysis(fieldContext.fieldName); if (canHighlight(fieldType) == false) { throw new IllegalArgumentException("the field [" + fieldContext.fieldName + @@ -95,10 +96,10 @@ public HighlightField highlight(FieldHighlightContext fieldContext) throws IOExc fragListBuilder = new SingleFragListBuilder(); if (!forceSource && fieldType.isStored()) { - fragmentsBuilder = new SimpleFragmentsBuilder(fieldType, field.fieldOptions().preTags(), + fragmentsBuilder = new SimpleFragmentsBuilder(fieldType, fixBrokenAnalysis, field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner); } else { - fragmentsBuilder = new SourceSimpleFragmentsBuilder(fieldType, hitContext.sourceLookup(), + fragmentsBuilder = new SourceSimpleFragmentsBuilder(fieldType, fixBrokenAnalysis, hitContext.sourceLookup(), field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner); } } else { @@ -109,16 +110,16 @@ public HighlightField highlight(FieldHighlightContext fieldContext) throws IOExc fragmentsBuilder = new ScoreOrderFragmentsBuilder(field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner); } else { - fragmentsBuilder = new SourceScoreOrderFragmentsBuilder(fieldType, hitContext.sourceLookup(), + fragmentsBuilder = new SourceScoreOrderFragmentsBuilder(fieldType, fixBrokenAnalysis, hitContext.sourceLookup(), field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner); } } else { if (!forceSource && fieldType.isStored()) { - fragmentsBuilder = new SimpleFragmentsBuilder(fieldType, field.fieldOptions().preTags(), + fragmentsBuilder = new SimpleFragmentsBuilder(fieldType, fixBrokenAnalysis, field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner); } else { fragmentsBuilder = - new SourceSimpleFragmentsBuilder(fieldType, hitContext.sourceLookup(), + new SourceSimpleFragmentsBuilder(fieldType, fixBrokenAnalysis, hitContext.sourceLookup(), field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner); } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java index d896d3a9d922d..d485e25d3ce45 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java @@ -20,7 +20,6 @@ package org.elasticsearch.search.fetch.subphase.highlight; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.document.Field; import org.apache.lucene.search.vectorhighlight.FastVectorHighlighter; import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo; import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo.SubInfo; @@ -29,9 +28,7 @@ import org.elasticsearch.index.analysis.AnalyzerComponentsProvider; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.analysis.TokenFilterFactory; -import org.elasticsearch.index.mapper.MappedFieldType; -import java.util.Comparator; import java.util.List; /** @@ -47,23 +44,19 @@ private FragmentBuilderHelper() { * Fixes problems with broken analysis chains if positions and offsets are messed up that can lead to * {@link StringIndexOutOfBoundsException} in the {@link FastVectorHighlighter} */ - public static WeightedFragInfo fixWeightedFragInfo(MappedFieldType fieldType, Field[] values, WeightedFragInfo fragInfo) { + public static WeightedFragInfo fixWeightedFragInfo(WeightedFragInfo fragInfo) { assert fragInfo != null : "FragInfo must not be null"; - assert fieldType.name().equals(values[0].name()) : "Expected MappedFieldType for field " + values[0].name(); - if (!fragInfo.getSubInfos().isEmpty() && containsBrokenAnalysis(fieldType.indexAnalyzer())) { + if (!fragInfo.getSubInfos().isEmpty()) { /* This is a special case where broken analysis like WDF is used for term-vector creation at index-time * which can potentially mess up the offsets. To prevent a SAIIOBException we need to resort * the fragments based on their offsets rather than using solely the positions as it is done in * the FastVectorHighlighter. Yet, this is really a lucene problem and should be fixed in lucene rather * than in this hack... aka. "we are are working on in!" */ final List subInfos = fragInfo.getSubInfos(); - CollectionUtil.introSort(subInfos, new Comparator() { - @Override - public int compare(SubInfo o1, SubInfo o2) { - int startOffset = o1.getTermsOffsets().get(0).getStartOffset(); - int startOffset2 = o2.getTermsOffsets().get(0).getStartOffset(); - return FragmentBuilderHelper.compare(startOffset, startOffset2); - } + CollectionUtil.introSort(subInfos, (o1, o2) -> { + int startOffset = o1.getTermsOffsets().get(0).getStartOffset(); + int startOffset2 = o2.getTermsOffsets().get(0).getStartOffset(); + return compare(startOffset, startOffset2); }); return new WeightedFragInfo(Math.min(fragInfo.getSubInfos().get(0).getTermsOffsets().get(0).getStartOffset(), fragInfo.getStartOffset()), fragInfo.getEndOffset(), subInfos, fragInfo.getTotalBoost()); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SimpleFragmentsBuilder.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SimpleFragmentsBuilder.java index 851d916cde7e2..70afd6c365ab9 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SimpleFragmentsBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SimpleFragmentsBuilder.java @@ -29,20 +29,26 @@ * that corrects offsets for broken analysis chains. */ public class SimpleFragmentsBuilder extends org.apache.lucene.search.vectorhighlight.SimpleFragmentsBuilder { + protected final MappedFieldType fieldType; + private final boolean fixBrokenAnalysis; public SimpleFragmentsBuilder(MappedFieldType fieldType, + boolean fixBrokenAnalysis, String[] preTags, String[] postTags, BoundaryScanner boundaryScanner) { super(preTags, postTags, boundaryScanner); this.fieldType = fieldType; + this.fixBrokenAnalysis = fixBrokenAnalysis; } @Override protected String makeFragment( StringBuilder buffer, int[] index, Field[] values, WeightedFragInfo fragInfo, - String[] preTags, String[] postTags, Encoder encoder ){ - WeightedFragInfo weightedFragInfo = FragmentBuilderHelper.fixWeightedFragInfo(fieldType, values, fragInfo); - return super.makeFragment(buffer, index, values, weightedFragInfo, preTags, postTags, encoder); + String[] preTags, String[] postTags, Encoder encoder) { + if (fixBrokenAnalysis) { + fragInfo = FragmentBuilderHelper.fixWeightedFragInfo(fragInfo); + } + return super.makeFragment(buffer, index, values, fragInfo, preTags, postTags, encoder); } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SourceScoreOrderFragmentsBuilder.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SourceScoreOrderFragmentsBuilder.java index dd8df92a8670c..aa62aaf9813f3 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SourceScoreOrderFragmentsBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SourceScoreOrderFragmentsBuilder.java @@ -35,8 +35,10 @@ public class SourceScoreOrderFragmentsBuilder extends ScoreOrderFragmentsBuilder private final MappedFieldType fieldType; private final SourceLookup sourceLookup; + private final boolean fixBrokenAnalysis; public SourceScoreOrderFragmentsBuilder(MappedFieldType fieldType, + boolean fixBrokenAnalysis, SourceLookup sourceLookup, String[] preTags, String[] postTags, @@ -44,6 +46,7 @@ public SourceScoreOrderFragmentsBuilder(MappedFieldType fieldType, super(preTags, postTags, boundaryScanner); this.fieldType = fieldType; this.sourceLookup = sourceLookup; + this.fixBrokenAnalysis = fixBrokenAnalysis; } @Override @@ -59,8 +62,10 @@ protected Field[] getFields(IndexReader reader, int docId, String fieldName) thr @Override protected String makeFragment( StringBuilder buffer, int[] index, Field[] values, WeightedFragInfo fragInfo, - String[] preTags, String[] postTags, Encoder encoder ){ - WeightedFragInfo weightedFragInfo = FragmentBuilderHelper.fixWeightedFragInfo(fieldType, values, fragInfo); - return super.makeFragment(buffer, index, values, weightedFragInfo, preTags, postTags, encoder); + String[] preTags, String[] postTags, Encoder encoder) { + if (fixBrokenAnalysis) { + fragInfo = FragmentBuilderHelper.fixWeightedFragInfo(fragInfo); + } + return super.makeFragment(buffer, index, values, fragInfo, preTags, postTags, encoder); } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SourceSimpleFragmentsBuilder.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SourceSimpleFragmentsBuilder.java index 59a90c0749522..ec5542e1c407b 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SourceSimpleFragmentsBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SourceSimpleFragmentsBuilder.java @@ -33,11 +33,12 @@ public class SourceSimpleFragmentsBuilder extends SimpleFragmentsBuilder { private final SourceLookup sourceLookup; public SourceSimpleFragmentsBuilder(MappedFieldType fieldType, + boolean fixBrokenAnalysis, SourceLookup sourceLookup, String[] preTags, String[] postTags, BoundaryScanner boundaryScanner) { - super(fieldType, preTags, postTags, boundaryScanner); + super(fieldType, fixBrokenAnalysis, preTags, postTags, boundaryScanner); this.sourceLookup = sourceLookup; } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index d0b14621ae98b..1bb255bac15b5 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.action.admin.indices.rollover; -import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; @@ -59,7 +58,6 @@ import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; @@ -556,15 +554,16 @@ public void testRolloverClusterStateForDataStream() throws Exception { DateFieldMapper dateFieldMapper = new DateFieldMapper.Builder("@timestamp", DateFieldMapper.Resolution.MILLISECONDS, null, false, Version.CURRENT) .build(builderContext); - MetadataFieldMapper mockedTimestampField = mock(MetadataFieldMapper.class); - when(mockedTimestampField.name()).thenReturn("_data_stream_timestamp"); MappedFieldType mockedTimestampFieldType = mock(MappedFieldType.class); when(mockedTimestampFieldType.name()).thenReturn("_data_stream_timestamp"); - when(mockedTimestampField.fieldType()).thenReturn(mockedTimestampFieldType); - when(mockedTimestampField.copyTo()).thenReturn(FieldMapper.CopyTo.empty()); - when(mockedTimestampField.multiFields()).thenReturn(FieldMapper.MultiFields.empty()); + MetadataFieldMapper mockedTimestampField = new MetadataFieldMapper(mockedTimestampFieldType) { + @Override + protected String contentType() { + return null; + } + }; MappingLookup mappingLookup = - new MappingLookup(List.of(mockedTimestampField, dateFieldMapper), List.of(), List.of(), 0, new StandardAnalyzer()); + new MappingLookup(List.of(mockedTimestampField, dateFieldMapper), List.of(), List.of(), 0); ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool); Environment env = mock(Environment.class); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java index 300ee18cfbc51..379ea2079b756 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java @@ -126,7 +126,7 @@ public void testDefaultConfiguration() throws IOException { assertThat(fieldMapper, instanceOf(CompletionFieldMapper.class)); MappedFieldType completionFieldType = ((CompletionFieldMapper) fieldMapper).fieldType(); - NamedAnalyzer indexAnalyzer = completionFieldType.indexAnalyzer(); + NamedAnalyzer indexAnalyzer = (NamedAnalyzer) ((CompletionFieldMapper) fieldMapper).indexAnalyzers().values().iterator().next(); assertThat(indexAnalyzer.name(), equalTo("simple")); assertThat(indexAnalyzer.analyzer(), instanceOf(CompletionAnalyzer.class)); CompletionAnalyzer analyzer = (CompletionAnalyzer) indexAnalyzer.analyzer(); @@ -155,7 +155,7 @@ public void testCompletionAnalyzerSettings() throws Exception { assertThat(fieldMapper, instanceOf(CompletionFieldMapper.class)); MappedFieldType completionFieldType = ((CompletionFieldMapper) fieldMapper).fieldType(); - NamedAnalyzer indexAnalyzer = completionFieldType.indexAnalyzer(); + NamedAnalyzer indexAnalyzer = (NamedAnalyzer) ((CompletionFieldMapper) fieldMapper).indexAnalyzers().values().iterator().next(); assertThat(indexAnalyzer.name(), equalTo("simple")); assertThat(indexAnalyzer.analyzer(), instanceOf(CompletionAnalyzer.class)); CompletionAnalyzer analyzer = (CompletionAnalyzer) indexAnalyzer.analyzer(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java index 66766039e7c61..829550a5439e2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java @@ -48,10 +48,10 @@ private static class FakeAnalyzer extends Analyzer { protected TokenStreamComponents createComponents(String fieldName) { Tokenizer tokenizer = new Tokenizer() { boolean incremented = false; - CharTermAttribute term = addAttribute(CharTermAttribute.class); + final CharTermAttribute term = addAttribute(CharTermAttribute.class); @Override - public boolean incrementToken() throws IOException { + public boolean incrementToken() { if (incremented) { return false; } @@ -84,8 +84,13 @@ public String typeName() { static class FakeFieldMapper extends FieldMapper { - FakeFieldMapper(FakeFieldType fieldType) { - super(fieldType.name(), fieldType, MultiFields.empty(), CopyTo.empty()); + final String indexedValue; + + FakeFieldMapper(FakeFieldType fieldType, String indexedValue) { + super(fieldType.name(), fieldType, + new NamedAnalyzer("fake", AnalyzerScope.INDEX, new FakeAnalyzer(indexedValue)), + MultiFields.empty(), CopyTo.empty()); + this.indexedValue = indexedValue; } @Override @@ -105,23 +110,20 @@ public Builder getMergeBuilder() { public void testAnalyzers() throws IOException { FakeFieldType fieldType1 = new FakeFieldType("field1"); - fieldType1.setIndexAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.INDEX, new FakeAnalyzer("index"))); - FieldMapper fieldMapper1 = new FakeFieldMapper(fieldType1); + FieldMapper fieldMapper1 = new FakeFieldMapper(fieldType1, "index1"); FakeFieldType fieldType2 = new FakeFieldType("field2"); - FieldMapper fieldMapper2 = new FakeFieldMapper(fieldType2); - - Analyzer defaultIndex = new FakeAnalyzer("default_index"); + FieldMapper fieldMapper2 = new FakeFieldMapper(fieldType2, "index2"); MappingLookup mappingLookup = new MappingLookup( Arrays.asList(fieldMapper1, fieldMapper2), Collections.emptyList(), Collections.emptyList(), - 0, defaultIndex); - - assertAnalyzes(mappingLookup.indexAnalyzer(), "field1", "index"); + 0); - assertAnalyzes(mappingLookup.indexAnalyzer(), "field2", "default_index"); + assertAnalyzes(mappingLookup.indexAnalyzer(), "field1", "index1"); + assertAnalyzes(mappingLookup.indexAnalyzer(), "field2", "index2"); + expectThrows(IllegalArgumentException.class, () -> mappingLookup.indexAnalyzer().tokenStream("field3", "blah")); } private void assertAnalyzes(Analyzer analyzer, String field, String output) throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java index 1d8f88d9aa03f..a3825404e3f0b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java @@ -68,7 +68,6 @@ public static final class FakeStringFieldType extends StringFieldType { private FakeStringFieldType(String name, boolean stored, TextSearchInfo textSearchInfo) { super(name, true, stored, true, textSearchInfo, Collections.emptyMap()); - setIndexAnalyzer(Lucene.STANDARD_ANALYZER); } @Override @@ -84,7 +83,7 @@ public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchL protected FakeStringFieldMapper(MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo) { - super(mappedFieldType.name(), mappedFieldType, multiFields, copyTo); + super(mappedFieldType.name(), mappedFieldType, Lucene.STANDARD_ANALYZER, multiFields, copyTo); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java index a25ebf4fe2407..0351d8dd7b75f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java @@ -21,7 +21,6 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Explicit; -import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; @@ -43,7 +42,7 @@ public void testDuplicateFieldAliasAndObject() { new MappingLookup( Collections.emptyList(), singletonList(objectMapper), - singletonList(aliasMapper), 0, Lucene.STANDARD_ANALYZER)); + singletonList(aliasMapper), 0)); assertEquals("Alias [some.path] is defined both as an object and an alias", e.getMessage()); } @@ -56,7 +55,7 @@ public void testDuplicateFieldAliasAndConcreteField() { new MappingLookup( Arrays.asList(field, invalidField), emptyList(), - singletonList(invalidAlias), 0, Lucene.STANDARD_ANALYZER)); + singletonList(invalidAlias), 0)); assertEquals("Alias [invalid] is defined both as an alias and a concrete field", e.getMessage()); } @@ -69,7 +68,7 @@ public void testAliasThatRefersToAlias() { MappingLookup mappers = new MappingLookup( singletonList(field), emptyList(), - Arrays.asList(alias, invalidAlias), 0, Lucene.STANDARD_ANALYZER); + Arrays.asList(alias, invalidAlias), 0); alias.validate(mappers); MapperParsingException e = expectThrows(MapperParsingException.class, () -> { @@ -87,7 +86,7 @@ public void testAliasThatRefersToItself() { MappingLookup mappers = new MappingLookup( emptyList(), emptyList(), - singletonList(invalidAlias), 0, null); + singletonList(invalidAlias), 0); invalidAlias.validate(mappers); }); @@ -102,7 +101,7 @@ public void testAliasWithNonExistentPath() { MappingLookup mappers = new MappingLookup( emptyList(), emptyList(), - singletonList(invalidAlias), 0, Lucene.STANDARD_ANALYZER); + singletonList(invalidAlias), 0); invalidAlias.validate(mappers); }); @@ -118,7 +117,7 @@ public void testFieldAliasWithNestedScope() { singletonList(createFieldMapper("nested", "field")), singletonList(objectMapper), singletonList(aliasMapper), - 0, Lucene.STANDARD_ANALYZER); + 0); aliasMapper.validate(mappers); } @@ -130,7 +129,7 @@ public void testFieldAliasWithDifferentObjectScopes() { List.of(createFieldMapper("object1", "field")), List.of(createObjectMapper("object1"), createObjectMapper("object2")), singletonList(aliasMapper), - 0, Lucene.STANDARD_ANALYZER); + 0); aliasMapper.validate(mappers); } @@ -143,7 +142,7 @@ public void testFieldAliasWithNestedTarget() { singletonList(createFieldMapper("nested", "field")), Collections.singletonList(objectMapper), singletonList(aliasMapper), - 0, Lucene.STANDARD_ANALYZER); + 0); aliasMapper.validate(mappers); }); @@ -161,7 +160,7 @@ public void testFieldAliasWithDifferentNestedScopes() { singletonList(createFieldMapper("nested1", "field")), List.of(createNestedObjectMapper("nested1"), createNestedObjectMapper("nested2")), singletonList(aliasMapper), - 0, Lucene.STANDARD_ANALYZER); + 0); aliasMapper.validate(mappers); }); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 3b14cd2873804..205ea95c47dc0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -774,7 +774,7 @@ protected TokenStreamComponents createComponents(String fieldName) { public void testIndexPrefixMapping() throws IOException { { - DocumentMapper mapper = createDocumentMapper( + MapperService ms = createMapperService( fieldMapping( b -> b.field("type", "text") .field("analyzer", "standard") @@ -785,11 +785,13 @@ public void testIndexPrefixMapping() throws IOException { ) ); - assertThat(mapper.mappers().getMapper("field._index_prefix").toString(), containsString("prefixChars=2:10")); + assertThat(ms.documentMapper().mappers().getMapper("field._index_prefix").toString(), containsString("prefixChars=2:10")); - ParsedDocument doc = mapper.parse(source(b -> b.field("field", "Some English text that is going to be very useful"))); + ParsedDocument doc + = ms.documentMapper().parse(source(b -> b.field("field", "Some English text that is going to be very useful"))); IndexableField[] fields = doc.rootDoc().getFields("field._index_prefix"); assertEquals(1, fields.length); + withLuceneIndex(ms, iw -> iw.addDocument(doc.rootDoc()), ir -> {}); // check we can index } { @@ -867,6 +869,10 @@ public void testFastPhrasePrefixes() throws IOException { } b.endObject(); })); + + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field("synfield", "some text which we will index"))); + withLuceneIndex(mapperService, iw -> iw.addDocument(doc.rootDoc()), ir -> {}); // check indexing + QueryShardContext queryShardContext = createQueryShardContext(mapperService); { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java index 8ba1f23dcfe5f..2b30c2fac063e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java @@ -36,7 +36,6 @@ import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.lucene.BytesRefs; -import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType; @@ -165,7 +164,6 @@ public void testIndexPrefixes() { public void testFetchSourceValue() throws IOException { TextFieldType fieldType = createFieldType(); - fieldType.setIndexAnalyzer(Lucene.STANDARD_ANALYZER); assertEquals(List.of("value"), fetchSourceValue(fieldType, "value")); assertEquals(List.of("42"), fetchSourceValue(fieldType, 42L)); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java index e992eab92bf04..1c8b469b907f2 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java @@ -37,8 +37,6 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.index.analysis.AnalyzerScope; -import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.BinaryFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; @@ -108,9 +106,8 @@ protected Map getFieldAliases(MappedFieldType... fieldT public void testSignificance() throws IOException { TextFieldType textFieldType = new TextFieldType("text"); textFieldType.setFielddata(true); - textFieldType.setIndexAnalyzer(new NamedAnalyzer("my_analyzer", AnalyzerScope.GLOBAL, new StandardAnalyzer())); - IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); + IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment @@ -253,9 +250,8 @@ public void testNumericSignificance() throws IOException { public void testUnmapped() throws IOException { TextFieldType textFieldType = new TextFieldType("text"); textFieldType.setFielddata(true); - textFieldType.setIndexAnalyzer(new NamedAnalyzer("my_analyzer", AnalyzerScope.GLOBAL, new StandardAnalyzer())); - IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); + IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, indexWriterConfig)) { @@ -321,9 +317,8 @@ public void testRangeField() throws IOException { public void testFieldAlias() throws IOException { TextFieldType textFieldType = new TextFieldType("text"); textFieldType.setFielddata(true); - textFieldType.setIndexAnalyzer(new NamedAnalyzer("my_analyzer", AnalyzerScope.GLOBAL, new StandardAnalyzer())); - IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); + IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java index 84a732c038610..090990c2f6092 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java @@ -34,10 +34,7 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.index.analysis.AnalyzerScope; -import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.BinaryFieldMapper; -import org.elasticsearch.index.mapper.GeoPointFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType; @@ -79,14 +76,16 @@ protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldTy @Override protected List getSupportedValuesSourceTypes() { - return List.of(CoreValuesSourceType.BYTES); + return List.of( + CoreValuesSourceType.BOOLEAN, + CoreValuesSourceType.BYTES + ); } @Override protected List unsupportedMappedFieldTypes() { return List.of( - BinaryFieldMapper.CONTENT_TYPE, // binary fields are not supported because they do not have analyzers - GeoPointFieldMapper.CONTENT_TYPE // geopoint fields cannot use term queries + BinaryFieldMapper.CONTENT_TYPE // binary fields are not supported because they do not have analyzers ); } @@ -95,9 +94,8 @@ protected List unsupportedMappedFieldTypes() { */ public void testSignificance() throws IOException { TextFieldType textFieldType = new TextFieldType("text"); - textFieldType.setIndexAnalyzer(new NamedAnalyzer("my_analyzer", AnalyzerScope.GLOBAL, new StandardAnalyzer())); - IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); + IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, indexWriterConfig)) { @@ -146,9 +144,8 @@ public void testSignificance() throws IOException { */ public void testIncludeExcludes() throws IOException { TextFieldType textFieldType = new TextFieldType("text"); - textFieldType.setIndexAnalyzer(new NamedAnalyzer("my_analyzer", AnalyzerScope.GLOBAL, new StandardAnalyzer())); - - IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); + + IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, indexWriterConfig)) { @@ -159,7 +156,7 @@ public void testIncludeExcludes() throws IOException { try (IndexReader reader = DirectoryReader.open(w)) { assertEquals("test expects a single segment", 1, reader.leaves().size()); IndexSearcher searcher = new IndexSearcher(reader); - + // Inclusive of values { SignificantTextAggregationBuilder sigAgg = new SignificantTextAggregationBuilder("sig_text", "text"). @@ -176,7 +173,7 @@ public void testIncludeExcludes() throws IOException { assertNull(terms.getBucketByKey("even")); assertNotNull(terms.getBucketByKey("duplicate")); assertTrue(AggregationInspectionHelper.hasValue(sampler)); - + } // Exclusive of values { @@ -192,18 +189,16 @@ public void testIncludeExcludes() throws IOException { SignificantTerms terms = sampler.getAggregations().get("sig_text"); assertNotNull(terms.getBucketByKey("even")); - assertNull(terms.getBucketByKey("duplicate")); + assertNull(terms.getBucketByKey("duplicate")); assertTrue(AggregationInspectionHelper.hasValue(sampler)); - + } } } - } - - + } + public void testMissingField() throws IOException { TextFieldType textFieldType = new TextFieldType("text"); - textFieldType.setIndexAnalyzer(new NamedAnalyzer("my_analyzer", AnalyzerScope.GLOBAL, new StandardAnalyzer())); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); indexWriterConfig.setMaxBufferedDocs(100); @@ -222,7 +217,7 @@ public void testMissingField() throws IOException { try (IndexReader reader = DirectoryReader.open(w)) { IndexSearcher searcher = new IndexSearcher(reader); - + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> searchAndReduce(searcher, new TermQuery(new Term("text", "odd")), aggBuilder, textFieldType)); assertThat(e.getMessage(), equalTo("Field [this_field_does_not_exist] does not exist, SignificantText " @@ -230,12 +225,11 @@ public void testMissingField() throws IOException { } } } - + public void testFieldAlias() throws IOException { TextFieldType textFieldType = new TextFieldType("text"); - textFieldType.setIndexAnalyzer(new NamedAnalyzer("my_analyzer", AnalyzerScope.GLOBAL, new StandardAnalyzer())); - IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); + IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, indexWriterConfig)) { @@ -284,9 +278,8 @@ public void testFieldAlias() throws IOException { public void testInsideTermsAgg() throws IOException { TextFieldType textFieldType = new TextFieldType("text"); - textFieldType.setIndexAnalyzer(new NamedAnalyzer("my_analyzer", AnalyzerScope.GLOBAL, new StandardAnalyzer())); - IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); + IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, indexWriterConfig)) { @@ -343,9 +336,8 @@ private void indexDocuments(IndexWriter writer) throws IOException { */ public void testSignificanceOnTextArrays() throws IOException { TextFieldType textFieldType = new TextFieldType("text"); - textFieldType.setIndexAnalyzer(new NamedAnalyzer("my_analyzer", AnalyzerScope.GLOBAL, new StandardAnalyzer())); - IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); + IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, indexWriterConfig)) { diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 70a197f7619aa..0e4ebf82e88c7 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -280,6 +280,7 @@ public boolean shouldCache(Query query) { MapperService mapperService = mapperServiceMock(); when(mapperService.getIndexSettings()).thenReturn(indexSettings); when(mapperService.hasNested()).thenReturn(false); + when(mapperService.indexAnalyzer()).thenReturn(new StandardAnalyzer()); // for significant text QueryShardContext queryShardContext = queryShardContextMock(contextIndexSearcher, mapperService, indexSettings, circuitBreakerService, bigArrays); when(searchContext.getQueryShardContext()).thenReturn(queryShardContext); diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java index 0d8d4d5bd93d3..fe0ba9c87da7b 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java +++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java @@ -166,7 +166,6 @@ public KeyedFlattenedFieldType(String name, boolean indexed, boolean hasDocValue super(name, indexed, false, hasDocValues, splitQueriesOnWhitespace ? TextSearchInfo.WHITESPACE_MATCH_ONLY : TextSearchInfo.SIMPLE_MATCH_ONLY, meta); - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); this.key = key; } @@ -379,7 +378,6 @@ public RootFlattenedFieldType(String name, boolean indexed, boolean hasDocValues super(name, indexed, false, hasDocValues, splitQueriesOnWhitespace ? TextSearchInfo.WHITESPACE_MATCH_ONLY : TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.splitQueriesOnWhitespace = splitQueriesOnWhitespace; - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); } @Override @@ -414,7 +412,7 @@ public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchL private FlattenedFieldMapper(String simpleName, MappedFieldType mappedFieldType, Builder builder) { - super(simpleName, mappedFieldType, CopyTo.empty()); + super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, CopyTo.empty()); this.builder = builder; this.fieldParser = new FlattenedFieldParser(mappedFieldType.name(), keyedFieldName(), mappedFieldType, builder.depthLimit.get(), builder.ignoreAbove.get(), builder.nullValue.get()); diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index afa53e4f493b7..b2406283dd903 100644 --- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -122,7 +122,6 @@ public static final class VersionStringFieldType extends TermBasedFieldType { private VersionStringFieldType(String name, FieldType fieldType, Map meta) { super(name, true, false, true, new TextSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), meta); - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); } @Override @@ -309,7 +308,7 @@ private VersionStringFieldMapper( MultiFields multiFields, CopyTo copyTo ) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, multiFields, copyTo); this.fieldType = fieldType; } diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java index 156aca84b9cd7..720a2f4a566e5 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java @@ -208,16 +208,6 @@ public void parse(ParseContext context) throws IOException { context.doc().addWithKey(fieldType().name(), field); } - @Override - protected boolean indexedByDefault() { - return false; - } - - @Override - protected boolean docValuesByDefault() { - return true; - } - @Override protected void parseCreateField(ParseContext context) { throw new AssertionError("parse is implemented directly"); diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index c56ebfb41a871..0bbe89e876183 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -248,14 +248,15 @@ public static final class WildcardFieldType extends MappedFieldType { private final String nullValue; private final int ignoreAbove; + private final NamedAnalyzer analyzer; private WildcardFieldType(String name, String nullValue, int ignoreAbove, Version version, Map meta) { super(name, true, false, true, Defaults.TEXT_SEARCH_INFO, meta); if (version.onOrAfter(Version.V_7_10_0)) { - setIndexAnalyzer(WILDCARD_ANALYZER_7_10); + this.analyzer = WILDCARD_ANALYZER_7_10; } else { - setIndexAnalyzer(WILDCARD_ANALYZER_7_9); + this.analyzer = WILDCARD_ANALYZER_7_9; } this.nullValue = nullValue; this.ignoreAbove = ignoreAbove; @@ -639,7 +640,7 @@ protected void getNgramTokens(Set tokens, String fragment) { return; } // Break fragment into multiple Ngrams - TokenStream tokenizer = indexAnalyzer().tokenStream(name(), fragment); + TokenStream tokenizer = analyzer.tokenStream(name(), fragment); CharTermAttribute termAtt = tokenizer.addAttribute(CharTermAttribute.class); int foundTokens = 0; try { @@ -658,7 +659,7 @@ protected void getNgramTokens(Set tokens, String fragment) { if (foundTokens == 0 && fragment.length() > 0) { // fragment must have been less than NGRAM_SIZE - add a placeholder which may be used in a prefix query e.g. ab* fragment = toLowerCase(fragment); - if (indexAnalyzer() == WILDCARD_ANALYZER_7_10) { + if (analyzer == WILDCARD_ANALYZER_7_10) { fragment = PunctuationFoldingFilter.normalize(fragment); } tokens.add(fragment); @@ -784,7 +785,7 @@ public Query fuzzyQuery( } } // Tokenize all content after the prefix - TokenStream tokenizer = indexAnalyzer().tokenStream(name(), postPrefixString); + TokenStream tokenizer = analyzer.tokenStream(name(), postPrefixString); CharTermAttribute termAtt = tokenizer.addAttribute(CharTermAttribute.class); ArrayList postPrefixTokens = new ArrayList<>(); String firstToken = null; @@ -928,10 +929,10 @@ protected String parseSourceValue(Object value) { private final FieldType ngramFieldType; private final Version indexVersionCreated; - private WildcardFieldMapper(String simpleName, MappedFieldType mappedFieldType, + private WildcardFieldMapper(String simpleName, WildcardFieldType mappedFieldType, int ignoreAbove, MultiFields multiFields, CopyTo copyTo, String nullValue, Version indexVersionCreated) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, mappedFieldType.analyzer, multiFields, copyTo); this.nullValue = nullValue; this.ignoreAbove = ignoreAbove; this.indexVersionCreated = indexVersionCreated;