Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move indexAnalyzer to FieldTypeLookup #63932

Closed
wants to merge 3 commits into from
Closed

Move indexAnalyzer to FieldTypeLookup #63932

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d8ce9ae
Move indexAnalyzer logic to FieldTypeLookup
javanna Oct 20, 2020
5d98316
iter
javanna Oct 20, 2020
d0eb59e
Merge branch 'master' into refactoring/index_analyzer_field_lookup
javanna Oct 20, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -470,8 +470,8 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
for (BytesReference document : documents) {
docs.add(docMapper.parse(new SourceToParse(context.index().getName(), "_temp_id", document, documentXContentType)));
}

FieldNameAnalyzer fieldNameAnalyzer = (FieldNameAnalyzer)docMapper.mappers().indexAnalyzer();
FieldNameAnalyzer fieldNameAnalyzer = docMapper.mappers().indexAnalyzer();
// Need to this custom impl because FieldNameAnalyzer is strict and the percolator sometimes isn't when
// 'index.percolator.map_unmapped_fields_as_string' is enabled:
Analyzer analyzer = new DelegatingAnalyzerWrapper(Analyzer.PER_FIELD_REUSE_STRATEGY) {
Expand Down
31 changes: 20 additions & 11 deletions server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,12 @@

package org.elasticsearch.index.mapper;

import org.apache.lucene.analysis.Analyzer;
import org.elasticsearch.common.collect.Iterators;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.index.analysis.FieldNameAnalyzer;

import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
Expand All @@ -33,7 +34,7 @@
/**
* An immutable container for looking up {@link MappedFieldType}s by their name.
*/
class FieldTypeLookup implements Iterable<MappedFieldType> {
final class FieldTypeLookup implements Iterable<MappedFieldType> {

private final Map<String, MappedFieldType> fullNameToFieldType = new HashMap<>();
private final Map<String, String> aliasToConcreteName = new HashMap<>();
Expand All @@ -47,18 +48,17 @@ class FieldTypeLookup implements Iterable<MappedFieldType> {
*/
private final Map<String, Set<String>> fieldToCopiedFields = new HashMap<>();
private final DynamicKeyFieldTypeLookup dynamicKeyLookup;

FieldTypeLookup() {
this(Collections.emptyList(), Collections.emptyList());
}
private final FieldNameAnalyzer indexAnalyzer;

FieldTypeLookup(Collection<FieldMapper> fieldMappers,
Collection<FieldAliasMapper> fieldAliasMappers) {
Collection<FieldAliasMapper> fieldAliasMappers,
Analyzer defaultIndexAnalyzer) {
Map<String, DynamicKeyFieldMapper> dynamicKeyMappers = new HashMap<>();

Map<String, Analyzer> indexAnalyzers = new HashMap<>();
for (FieldMapper fieldMapper : fieldMappers) {
String fieldName = fieldMapper.name();
MappedFieldType fieldType = fieldMapper.fieldType();
indexAnalyzers.put(fieldType.name(), fieldType.indexAnalyzer() != null ? fieldType.indexAnalyzer() : defaultIndexAnalyzer);
fullNameToFieldType.put(fieldType.name(), fieldType);
if (fieldMapper instanceof DynamicKeyFieldMapper) {
dynamicKeyMappers.put(fieldName, (DynamicKeyFieldMapper) fieldMapper);
Expand All @@ -82,12 +82,13 @@ class FieldTypeLookup implements Iterable<MappedFieldType> {
}

this.dynamicKeyLookup = new DynamicKeyFieldTypeLookup(dynamicKeyMappers, aliasToConcreteName);
this.indexAnalyzer = new FieldNameAnalyzer(indexAnalyzers);
}

/**
* Returns the mapped field type for the given field name.
*/
public MappedFieldType get(String field) {
MappedFieldType get(String field) {
String concreteField = aliasToConcreteName.getOrDefault(field, field);
MappedFieldType fieldType = fullNameToFieldType.get(concreteField);
if (fieldType != null) {
Expand All @@ -99,10 +100,18 @@ public MappedFieldType get(String field) {
return dynamicKeyLookup.get(field);
}

/**
* A smart analyzer used for indexing that takes into account specific analyzers configured
* per {@link FieldMapper}.
*/
FieldNameAnalyzer indexAnalyzer() {
return this.indexAnalyzer;
}

/**
* Returns a list of the full names of a simple match regex like pattern against full name and index name.
*/
public Set<String> simpleMatchToFullName(String pattern) {
Set<String> simpleMatchToFullName(String pattern) {
Set<String> fields = new HashSet<>();
for (MappedFieldType fieldType : this) {
if (Regex.simpleMatch(pattern, fieldType.name())) {
Expand All @@ -129,7 +138,7 @@ public Set<String> simpleMatchToFullName(String pattern) {
* should be a concrete field and *not* an alias.
* @return A set of paths in the _source that contain the field's values.
*/
public Set<String> sourcePaths(String field) {
Set<String> sourcePaths(String field) {
String resolvedField = field;
int lastDotIndex = field.lastIndexOf('.');
if (lastDotIndex > 0) {
Expand Down
27 changes: 8 additions & 19 deletions server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,16 +40,8 @@ public final class MappingLookup implements Iterable<Mapper> {
private final boolean hasNested;
private final FieldTypeLookup fieldTypeLookup;
private final int metadataFieldCount;
private final FieldNameAnalyzer indexAnalyzer;

private static void put(Map<String, Analyzer> analyzers, String key, Analyzer value, Analyzer defaultValue) {
if (value == null) {
value = defaultValue;
}
analyzers.put(key, value);
}

public static MappingLookup fromMapping(Mapping mapping, Analyzer defaultIndex) {
public static MappingLookup fromMapping(Mapping mapping, Analyzer defaultIndexAnalyzer) {
List<ObjectMapper> newObjectMappers = new ArrayList<>();
List<FieldMapper> newFieldMappers = new ArrayList<>();
List<FieldAliasMapper> newFieldAliasMappers = new ArrayList<>();
Expand All @@ -59,7 +51,8 @@ public static MappingLookup fromMapping(Mapping mapping, Analyzer defaultIndex)
}
}
collect(mapping.root, newObjectMappers, newFieldMappers, newFieldAliasMappers);
return new MappingLookup(newFieldMappers, newObjectMappers, newFieldAliasMappers, mapping.metadataMappers.length, defaultIndex);
return new MappingLookup(newFieldMappers, newObjectMappers, newFieldAliasMappers, mapping.metadataMappers.length,
defaultIndexAnalyzer);
}

private static void collect(Mapper mapper, Collection<ObjectMapper> objectMappers,
Expand Down Expand Up @@ -87,9 +80,8 @@ public MappingLookup(Collection<FieldMapper> mappers,
Collection<ObjectMapper> objectMappers,
Collection<FieldAliasMapper> aliasMappers,
int metadataFieldCount,
Analyzer defaultIndex) {
Analyzer defaultIndexAnalyzer) {
Map<String, Mapper> fieldMappers = new HashMap<>();
Map<String, Analyzer> indexAnalyzers = new HashMap<>();
Map<String, ObjectMapper> objects = new HashMap<>();

boolean hasNested = false;
Expand All @@ -110,8 +102,7 @@ public MappingLookup(Collection<FieldMapper> mappers,
if (fieldMappers.put(mapper.name(), mapper) != null) {
throw new MapperParsingException("Field [" + mapper.name() + "] is defined more than once");
}
MappedFieldType fieldType = mapper.fieldType();
put(indexAnalyzers, fieldType.name(), fieldType.indexAnalyzer(), defaultIndex);

}
this.metadataFieldCount = metadataFieldCount;

Expand All @@ -124,10 +115,8 @@ public MappingLookup(Collection<FieldMapper> mappers,
}
}

this.fieldTypeLookup = new FieldTypeLookup(mappers, aliasMappers);

this.fieldTypeLookup = new FieldTypeLookup(mappers, aliasMappers, defaultIndexAnalyzer);
this.fieldMappers = Collections.unmodifiableMap(fieldMappers);
this.indexAnalyzer = new FieldNameAnalyzer(indexAnalyzers);
this.objectMappers = Collections.unmodifiableMap(objects);
}

Expand All @@ -149,8 +138,8 @@ public FieldTypeLookup fieldTypes() {
* A smart analyzer used for indexing that takes into account specific analyzers configured
* per {@link FieldMapper}.
*/
public Analyzer indexAnalyzer() {
return this.indexAnalyzer;
public FieldNameAnalyzer indexAnalyzer() {
return this.fieldTypeLookup.indexAnalyzer();
}

@Override
Expand Down
16 changes: 9 additions & 7 deletions server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.elasticsearch.index.mapper;

import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.test.ESTestCase;

import java.util.Arrays;
Expand All @@ -33,7 +34,7 @@
public class FieldTypeLookupTests extends ESTestCase {

public void testEmpty() {
FieldTypeLookup lookup = new FieldTypeLookup();
FieldTypeLookup lookup = new FieldTypeLookup(Collections.emptyList(), Collections.emptyList(), null);
assertNull(lookup.get("foo"));
Collection<String> names = lookup.simpleMatchToFullName("foo");
assertNotNull(names);
Expand All @@ -45,7 +46,7 @@ public void testEmpty() {

public void testAddNewField() {
MockFieldMapper f = new MockFieldMapper("foo");
FieldTypeLookup lookup = new FieldTypeLookup(Collections.singletonList(f), emptyList());
FieldTypeLookup lookup = new FieldTypeLookup(Collections.singletonList(f), emptyList(), Lucene.KEYWORD_ANALYZER);
assertNull(lookup.get("bar"));
assertEquals(f.fieldType(), lookup.get("foo"));
assertEquals(1, size(lookup.iterator()));
Expand All @@ -55,7 +56,8 @@ public void testAddFieldAlias() {
MockFieldMapper field = new MockFieldMapper("foo");
FieldAliasMapper alias = new FieldAliasMapper("alias", "alias", "foo");

FieldTypeLookup lookup = new FieldTypeLookup(Collections.singletonList(field), Collections.singletonList(alias));
FieldTypeLookup lookup = new FieldTypeLookup(Collections.singletonList(field), Collections.singletonList(alias),
Lucene.KEYWORD_ANALYZER);

MappedFieldType aliasType = lookup.get("alias");
assertEquals(field.fieldType(), aliasType);
Expand All @@ -68,7 +70,7 @@ public void testSimpleMatchToFullName() {
FieldAliasMapper alias1 = new FieldAliasMapper("food", "food", "foo");
FieldAliasMapper alias2 = new FieldAliasMapper("barometer", "barometer", "bar");

FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(field1, field2), Arrays.asList(alias1, alias2));
FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(field1, field2), Arrays.asList(alias1, alias2), Lucene.KEYWORD_ANALYZER);

Collection<String> names = lookup.simpleMatchToFullName("b*");

Expand All @@ -88,7 +90,7 @@ public void testSourcePathWithMultiFields() {
.addMultiField(new MockFieldMapper.Builder("field.subfield2"))
.build(context);

FieldTypeLookup lookup = new FieldTypeLookup(singletonList(field), emptyList());
FieldTypeLookup lookup = new FieldTypeLookup(singletonList(field), emptyList(), Lucene.KEYWORD_ANALYZER);

assertEquals(Set.of("field"), lookup.sourcePaths("field"));
assertEquals(Set.of("field"), lookup.sourcePaths("field.subfield1"));
Expand All @@ -107,15 +109,15 @@ public void testSourcePathsWithCopyTo() {
.copyTo("field")
.build(context);

FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(field, otherField), emptyList());
FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(field, otherField), emptyList(), Lucene.KEYWORD_ANALYZER);

assertEquals(Set.of("other_field", "field"), lookup.sourcePaths("field"));
assertEquals(Set.of("other_field", "field"), lookup.sourcePaths("field.subfield1"));
}

public void testIteratorImmutable() {
MockFieldMapper f1 = new MockFieldMapper("foo");
FieldTypeLookup lookup = new FieldTypeLookup(Collections.singletonList(f1), emptyList());
FieldTypeLookup lookup = new FieldTypeLookup(Collections.singletonList(f1), emptyList(), Lucene.KEYWORD_ANALYZER);

try {
Iterator<MappedFieldType> itr = lookup.iterator();
Expand Down
11 changes: 6 additions & 5 deletions ...er-flattened/src/test/java/org/elasticsearch/index/mapper/FlatObjectFieldLookupTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
package org.elasticsearch.index.mapper;

import org.elasticsearch.Version;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.LeafFieldData;
Expand Down Expand Up @@ -41,7 +42,7 @@ public void testFieldTypeLookup() {
String fieldName = "object1.object2.field";
FlatObjectFieldMapper mapper = createFlatObjectMapper(fieldName);

FieldTypeLookup lookup = new FieldTypeLookup(singletonList(mapper), emptyList());
FieldTypeLookup lookup = new FieldTypeLookup(singletonList(mapper), emptyList(), Lucene.KEYWORD_ANALYZER);
assertEquals(mapper.fieldType(), lookup.get(fieldName));

String objectKey = "key1.key2";
Expand All @@ -62,7 +63,7 @@ public void testFieldTypeLookupWithAlias() {
String aliasName = "alias";
FieldAliasMapper alias = new FieldAliasMapper(aliasName, aliasName, fieldName);

FieldTypeLookup lookup = new FieldTypeLookup(singletonList(mapper), singletonList(alias));
FieldTypeLookup lookup = new FieldTypeLookup(singletonList(mapper), singletonList(alias), Lucene.KEYWORD_ANALYZER);
assertEquals(mapper.fieldType(), lookup.get(aliasName));

String objectKey = "key1.key2";
Expand All @@ -85,11 +86,11 @@ public void testFieldTypeLookupWithMultipleFields() {
FlatObjectFieldMapper mapper2 = createFlatObjectMapper(field2);
FlatObjectFieldMapper mapper3 = createFlatObjectMapper(field3);

FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(mapper1, mapper2), emptyList());
FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(mapper1, mapper2), emptyList(), Lucene.KEYWORD_ANALYZER);
assertNotNull(lookup.get(field1 + ".some.key"));
assertNotNull(lookup.get(field2 + ".some.key"));

lookup = new FieldTypeLookup(Arrays.asList(mapper1, mapper2, mapper3), emptyList());
lookup = new FieldTypeLookup(Arrays.asList(mapper1, mapper2, mapper3), emptyList(), Lucene.KEYWORD_ANALYZER);
assertNotNull(lookup.get(field1 + ".some.key"));
assertNotNull(lookup.get(field2 + ".some.key"));
assertNotNull(lookup.get(field3 + ".some.key"));
Expand Down Expand Up @@ -126,7 +127,7 @@ public void testFieldLookupIterator() {
MockFieldMapper mapper = new MockFieldMapper("foo");
FlatObjectFieldMapper flatObjectMapper = createFlatObjectMapper("object1.object2.field");

FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(mapper, flatObjectMapper), emptyList());
FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(mapper, flatObjectMapper), emptyList(), Lucene.KEYWORD_ANALYZER);

Set<String> fieldNames = new HashSet<>();
for (MappedFieldType fieldType : lookup) {
Expand Down