Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Respect the ignore_above option in field retrieval API. #57307

Merged
merged 1 commit into from
Jun 10, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -750,6 +750,11 @@ protected String parseSourceValue(Object value, String format) {
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
return value.toString();

String keywordValue = value.toString();
if (keywordValue.length() > ignoreAbove) {
return null;
}
return keywordValue;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -488,5 +488,12 @@ public void testParseSourceValue() {
assertEquals("value", mapper.parseSourceValue("value", null));
assertEquals("42", mapper.parseSourceValue(42L, null));
assertEquals("true", mapper.parseSourceValue(true, null));

ICUCollationKeywordFieldMapper ignoreAboveMapper = new ICUCollationKeywordFieldMapper.Builder("field")
.ignoreAbove(4)
.build(context);
assertNull(ignoreAboveMapper.parseSourceValue("value", null));
assertEquals("42", ignoreAboveMapper.parseSourceValue(42L, null));
assertEquals("true", ignoreAboveMapper.parseSourceValue(true, null));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -349,7 +349,9 @@ public List<?> lookupValues(SourceLookup lookup, @Nullable String format) {
List<?> sourceValues = sourceValue instanceof List ? (List<?>) sourceValue : List.of(sourceValue);
for (Object value : sourceValues) {
Object parsedValue = parseSourceValue(value, format);
values.add(parsedValue);
if (parsedValue != null) {
values.add(parsedValue);
}
}
}
return values;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -186,14 +186,6 @@ public Mapper.Builder<?> parse(String name, Map<String, Object> node, ParserCont
}
}

@Override
protected String parseSourceValue(Object value, String format) {
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
return value.toString();
}

public static final class KeywordFieldType extends StringFieldType {

private NamedAnalyzer normalizer = null;
Expand Down Expand Up @@ -382,6 +374,20 @@ protected void parseCreateField(ParseContext context) throws IOException {
context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue));
}
}

@Override
protected String parseSourceValue(Object value, String format) {
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}

String keywordValue = value.toString();
if (keywordValue.length() > ignoreAbove) {
return null;
}
return keywordValue;
}

@Override
protected String contentType() {
return CONTENT_TYPE;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -604,13 +604,20 @@ public void testMeta() throws Exception {
public void testParseSourceValue() {
Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
KeywordFieldMapper mapper = new KeywordFieldMapper.Builder("field").build(context);

KeywordFieldMapper mapper = new KeywordFieldMapper.Builder("field").build(context);
assertEquals("value", mapper.parseSourceValue("value", null));
assertEquals("42", mapper.parseSourceValue(42L, null));
assertEquals("true", mapper.parseSourceValue(true, null));

IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> mapper.parseSourceValue(true, "format"));
assertEquals("Field [field] of type [keyword] doesn't support formats.", e.getMessage());

KeywordFieldMapper ignoreAboveMapper = new KeywordFieldMapper.Builder("field")
.ignoreAbove(4)
.build(context);
assertNull(ignoreAboveMapper.parseSourceValue("value", null));
assertEquals("42", ignoreAboveMapper.parseSourceValue(42L, null));
assertEquals("true", ignoreAboveMapper.parseSourceValue(true, null));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,33 @@ public void testDateFormat() throws IOException {
assertThat(dateField.getValue(), equalTo("1990/12/29"));
}

public void testIgnoreAbove() throws IOException {
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject()
.startObject("properties")
.startObject("field")
.field("type", "keyword")
.field("ignore_above", 20)
.endObject()
.endObject()
.endObject();

IndexService indexService = createIndex("index", Settings.EMPTY, mapping);
MapperService mapperService = indexService.mapperService();

XContentBuilder source = XContentFactory.jsonBuilder().startObject()
.array("field", "value", "other_value", "really_really_long_value")
.endObject();
Map<String, DocumentField> fields = retrieveFields(mapperService, source, "field");
DocumentField field = fields.get("field");
assertThat(field.getValues().size(), equalTo(2));

source = XContentFactory.jsonBuilder().startObject()
.array("field", "really_really_long_value")
.endObject();
fields = retrieveFields(mapperService, source, "field");
assertFalse(fields.containsKey("field"));
}

public void testFieldAliases() throws IOException {
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject()
.startObject("properties")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -893,7 +893,12 @@ protected String parseSourceValue(Object value, String format) {
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
return value.toString();

String keywordValue = value.toString();
if (keywordValue.length() > ignoreAbove) {
return null;
}
return keywordValue;
}

// For internal use by Lucene only - used to define ngram index
Expand Down
Loading