From cdcb03a2ba243fffc6b6f038d4eaa44f869d118c Mon Sep 17 00:00:00 2001 From: kkewwei Date: Fri, 11 Oct 2024 08:49:02 +0800 Subject: [PATCH] Flat object field should delegate to keyword field for most query types Signed-off-by: kkewwei --- CHANGELOG.md | 1 + .../test/index/90_flat_object.yml | 32 - .../92_flat_object_support_doc_values.yml | 788 +++++++++++++ .../xcontent/JsonToStringXContentParser.java | 8 +- .../index/mapper/FlatObjectFieldMapper.java | 310 +++-- .../index/mapper/KeywordFieldMapper.java | 62 +- .../mapper/FlatObjectFieldMapperTests.java | 45 +- .../mapper/FlatObjectFieldTypeTests.java | 1002 ++++++++++++++++- 8 files changed, 1990 insertions(+), 258 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/index/92_flat_object_support_doc_values.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 929b3561cdbb0..357d830ddc123 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - New `phone` & `phone-search` analyzer + tokenizer ([#15915](https://github.com/opensearch-project/OpenSearch/pull/15915)) - Add _list/shards API as paginated alternate to _cat/shards ([#14641](https://github.com/opensearch-project/OpenSearch/pull/14641)) - Latency and Memory allocation improvements to Multi Term Aggregation queries ([#14993](https://github.com/opensearch-project/OpenSearch/pull/14993)) +- Flat object field use IndexOrDocValuesQuery to optimize query ([#14383](https://github.com/opensearch-project/OpenSearch/issues/14383)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml index e8da81d7bee41..2a469aa5ff04d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml @@ -671,38 +671,6 @@ teardown: - match: { error.root_cause.0.reason: "Mapping definition for [data] has unsupported parameters: [analyzer : standard]"} - match: { status: 400 } - # Wildcard Query with dot path. - - do: - catch: bad_request - search: - body: { - _source: true, - query: { - "wildcard": { - "catalog.title": "Mock*" - } - } - } - - match: { error.root_cause.0.type: "query_shard_exception" } - - match: { error.root_cause.0.reason: "Can only use wildcard queries on keyword and text fields - not on [catalog.title] which is of type [flat_object]"} - - match: { status: 400 } - - # Wildcard Query without dot path. - - do: - catch: bad_request - search: - body: { - _source: true, - query: { - "wildcard": { - "catalog": "Mock*" - } - } - } - - match: { error.root_cause.0.type: "query_shard_exception" } - - match: { error.root_cause.0.reason: "Can only use wildcard queries on keyword and text fields - not on [catalog] which is of type [flat_object]" } - - match: { status: 400 } - # Aggregation and Match Query with dot path. - do: catch: bad_request diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/92_flat_object_support_doc_values.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/92_flat_object_support_doc_values.yml new file mode 100644 index 0000000000000..9ec39660a4928 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/92_flat_object_support_doc_values.yml @@ -0,0 +1,788 @@ +--- +# The test setup includes: +# - Create flat_object mapping for flat_object_doc_values_test index +# - Index 9 example documents +# - Search tests about doc_values and index + +setup: + - skip: + version: " - 2.99.99" + reason: "introduced in 3.0.0 " + + - do: + indices.create: + index: flat_object_doc_values_test + body: + mappings: + properties: + issue: + properties: + labels: + type: "flat_object" + order: + type: "keyword" + + - do: + bulk: + refresh: true + body: | + {"index":{"_index":"flat_object_doc_values_test","_id":"0"}} + {"order":"order0","issue":{"labels":{"number":1,"name":"abc0","status":1}}} + {"index":{"_index":"flat_object_doc_values_test","_id":"1"}} + {"order":"order1","issue":{"labels":{"number":2,"name":"abc1","status":1}}} + {"index":{"_index":"flat_object_doc_values_test","_id":"2"}} + {"order":"order2","issue":{"labels":{"number":2,"name":"abc2","status":1}}} + {"index":{"_index":"flat_object_doc_values_test","_id":"3"}} + {"order":"order3","issue":{"labels":{"number":3,"name":"abc3","status":1}}} + {"index":{"_index":"flat_object_doc_values_test","_id":"4"}} + {"order":"order4","issue":{"labels":{"number":4,"name":"abc4","status":1}}} + {"index":{"_index":"flat_object_doc_values_test","_id":"5"}} + {"order":"order5","issue":{"labels":{"number":5,"name":"abc5","status":1}}} + {"index":{"_index":"flat_object_doc_values_test","_id":"6"}} + {"order":"order6","issue":{"labels":{"number":6,"name":"abc6","status":1}}} + {"index":{"_index":"flat_object_doc_values_test","_id":"7"}} + {"order":"order7","issue":{"labels":{"number":7,"name":"abc7","status":1}}} + {"index":{"_index":"flat_object_doc_values_test","_id":"8"}} + {"order":"order8","issue":{"labels":{"number":8,"name":"abc8","status":1}}} + +--- +# Delete Index when connection is teardown +teardown: + - do: + indices.delete: + index: flat_object_doc_values_test + +--- +"Supported queries": + - skip: + version: " - 2.99.99" + reason: "introduced in 3.0.0 " + + # Verify Document Count + - do: + search: + body: { + query: { + match_all: { } + } + } + + - length: { hits.hits: 9 } + + # Term Query with exact dot path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + term: { + issue.labels.status: 1 + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + issue.labels.name: "abc8" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Term Query with no path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + term: { + issue.labels: 1 + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + issue.labels: "abc8" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Terms Query with exact dot path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + terms: { + issue.labels.status: [0,1] + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + terms: { + issue.labels.name: ["abc8"] + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Terms Query with no path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + terms: { + issue.labels: [ 0,1 ] + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + terms: { + issue.labels.name: ["abc8"] + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Prefix Query with exact dot path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + prefix: { + issue.labels.name: "ab" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + prefix: { + issue.labels.name: "abc8" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Prefix Query with no path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + prefix: { + issue.labels: "ab" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + prefix: { + issue.labels: "abc8" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Regexp Query with exact dot path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + regexp: { + issue.labels.name: "ab.*" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + - match: { hits.hits.0._source.issue.labels.name: "abc8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + regexp: { + issue.labels.name: "a.*c8" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Regexp Query with no path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + regexp: { + issue.labels: "ab.*" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + regexp: { + issue.labels: "a.*c8" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Fuzzy Query with exact dot path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + fuzzy: { + issue.labels.name: { + value: "abcx", + fuzziness: 1 + } + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + fuzzy: { + issue.labels.name: { + value: "abc8", + fuzziness: 0 + } + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Fuzzy Query with no path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + fuzzy: { + issue.labels: { + value: "abcx", + fuzziness: 1 + } + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + fuzzy: { + issue.labels: { + value: "abc8", + fuzziness: 0 + } + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Range Query with exact dot path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + range: { + issue.labels.status: { + from: 0 + } + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + range: { + issue.labels.name: { + from: "abc8" + } + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Range Query with no path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + range: { + issue.labels: { + from: 0 + } + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + range: { + issue.labels: { + from: "abc8" + } + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Exists Query with exact dot path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + exists: { + field: "issue.labels.status" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Exists Query with no path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + exists: { + field: "issue.labels" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + + # Wildcard Query with exact dot path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + wildcard: { + issue.labels.name: "abc*" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + wildcard: { + issue.labels.name: "abc8*" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Wildcard Query with no path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + wildcard: { + issue.labels: "abc*" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + wildcard: { + issue.labels: "abc8*" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } diff --git a/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java b/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java index 95a8d9c9495f2..21270b4241b15 100644 --- a/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java +++ b/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java @@ -37,6 +37,10 @@ * @opensearch.internal */ public class JsonToStringXContentParser extends AbstractXContentParser { + public static final String VALUE_AND_PATH_SUFFIX = "._valueAndPath"; + public static final String VALUE_SUFFIX = "._value"; + public static final String DOT_SYMBOL = "."; + public static final String EQUAL_SYMBOL = "="; private final String fieldTypeName; private final XContentParser parser; @@ -50,10 +54,6 @@ public class JsonToStringXContentParser extends AbstractXContentParser { private final DeprecationHandler deprecationHandler; - private static final String VALUE_AND_PATH_SUFFIX = "._valueAndPath"; - private static final String VALUE_SUFFIX = "._value"; - private static final String EQUAL_SYMBOL = "="; - public JsonToStringXContentParser( NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, diff --git a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java index 738efcfafdca1..0ccdb40f9d33a 100644 --- a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java @@ -16,19 +16,16 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.FieldExistsQuery; import org.apache.lucene.search.MultiTermQuery; -import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.util.BytesRef; -import org.opensearch.OpenSearchException; import org.opensearch.Version; import org.opensearch.common.Nullable; import org.opensearch.common.collect.Iterators; import org.opensearch.common.lucene.Lucene; -import org.opensearch.common.lucene.search.AutomatonQueries; +import org.opensearch.common.unit.Fuzziness; import org.opensearch.common.xcontent.JsonToStringXContentParser; import org.opensearch.core.common.ParsingException; import org.opensearch.core.xcontent.DeprecationHandler; @@ -37,8 +34,8 @@ import org.opensearch.index.analysis.NamedAnalyzer; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; +import org.opensearch.index.mapper.KeywordFieldMapper.KeywordFieldType; import org.opensearch.index.query.QueryShardContext; -import org.opensearch.index.query.QueryShardException; import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.lookup.SearchLookup; @@ -52,7 +49,11 @@ import java.util.function.BiFunction; import java.util.function.Supplier; -import static org.opensearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; +import static org.opensearch.common.xcontent.JsonToStringXContentParser.DOT_SYMBOL; +import static org.opensearch.common.xcontent.JsonToStringXContentParser.EQUAL_SYMBOL; +import static org.opensearch.common.xcontent.JsonToStringXContentParser.VALUE_AND_PATH_SUFFIX; +import static org.opensearch.common.xcontent.JsonToStringXContentParser.VALUE_SUFFIX; +import static org.opensearch.index.mapper.FlatObjectFieldMapper.FlatObjectFieldType.getKeywordFieldType; /** * A field mapper for flat_objects. @@ -62,10 +63,6 @@ public final class FlatObjectFieldMapper extends DynamicKeyFieldMapper { public static final String CONTENT_TYPE = "flat_object"; - private static final String VALUE_AND_PATH_SUFFIX = "._valueAndPath"; - private static final String VALUE_SUFFIX = "._value"; - private static final String DOT_SYMBOL = "."; - private static final String EQUAL_SYMBOL = "="; /** * In flat_object field mapper, field type is similar to keyword field type @@ -86,7 +83,12 @@ public static class Defaults { @Override public MappedFieldType keyedFieldType(String key) { - return new FlatObjectFieldType(this.name() + DOT_SYMBOL + key, this.name()); + return new FlatObjectFieldType( + this.name() + DOT_SYMBOL + key, + this.name(), + (KeywordFieldType) valueFieldMapper.fieldType(), + (KeywordFieldType) valueAndPathFieldMapper.fieldType() + ); } /** @@ -111,20 +113,12 @@ public Builder(String name) { builder = this; } - private FlatObjectFieldType buildFlatObjectFieldType(BuilderContext context, FieldType fieldType) { - return new FlatObjectFieldType(buildFullName(context), fieldType); - } - /** * ValueFieldMapper is the subfield type for values in the Json. * use a {@link KeywordFieldMapper.KeywordField} */ - private ValueFieldMapper buildValueFieldMapper(BuilderContext context, FieldType fieldType, FlatObjectFieldType fft) { - String fullName = buildFullName(context); + private ValueFieldMapper buildValueFieldMapper(FieldType fieldType, KeywordFieldType valueFieldType) { FieldType vft = new FieldType(fieldType); - KeywordFieldMapper.KeywordFieldType valueFieldType = new KeywordFieldMapper.KeywordFieldType(fullName + VALUE_SUFFIX, vft); - - fft.setValueFieldType(valueFieldType); return new ValueFieldMapper(vft, valueFieldType); } @@ -132,27 +126,30 @@ private ValueFieldMapper buildValueFieldMapper(BuilderContext context, FieldType * ValueAndPathFieldMapper is the subfield type for path=value format in the Json. * also use a {@link KeywordFieldMapper.KeywordField} */ - private ValueAndPathFieldMapper buildValueAndPathFieldMapper(BuilderContext context, FieldType fieldType, FlatObjectFieldType fft) { - String fullName = buildFullName(context); + private ValueAndPathFieldMapper buildValueAndPathFieldMapper(FieldType fieldType, KeywordFieldType valueAndPathFieldType) { FieldType vft = new FieldType(fieldType); - KeywordFieldMapper.KeywordFieldType ValueAndPathFieldType = new KeywordFieldMapper.KeywordFieldType( - fullName + VALUE_AND_PATH_SUFFIX, - vft - ); - fft.setValueAndPathFieldType(ValueAndPathFieldType); - return new ValueAndPathFieldMapper(vft, ValueAndPathFieldType); + return new ValueAndPathFieldMapper(vft, valueAndPathFieldType); } @Override public FlatObjectFieldMapper build(BuilderContext context) { - FieldType fieldtype = new FieldType(Defaults.FIELD_TYPE); - FlatObjectFieldType fft = buildFlatObjectFieldType(context, fieldtype); + boolean isSearchable = true; + boolean hasDocValue = true; + KeywordFieldType valueFieldType = getKeywordFieldType(buildFullName(context), VALUE_SUFFIX, isSearchable, hasDocValue); + KeywordFieldType valueAndPathFieldType = getKeywordFieldType( + buildFullName(context), + VALUE_AND_PATH_SUFFIX, + isSearchable, + hasDocValue + ); + FlatObjectFieldType fft = new FlatObjectFieldType(buildFullName(context), null, valueFieldType, valueAndPathFieldType); + return new FlatObjectFieldMapper( name, Defaults.FIELD_TYPE, fft, - buildValueFieldMapper(context, fieldtype, fft), - buildValueAndPathFieldMapper(context, fieldtype, fft), + buildValueFieldMapper(Defaults.FIELD_TYPE, valueFieldType), + buildValueAndPathFieldMapper(Defaults.FIELD_TYPE, valueAndPathFieldType), CopyTo.empty(), this ); @@ -189,66 +186,70 @@ public static final class FlatObjectFieldType extends StringFieldType { private final String mappedFieldTypeName; - private KeywordFieldMapper.KeywordFieldType valueFieldType; + private final KeywordFieldType valueFieldType; - private KeywordFieldMapper.KeywordFieldType valueAndPathFieldType; - - public FlatObjectFieldType(String name, boolean isSearchable, boolean hasDocValues, Map meta) { - super(name, isSearchable, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); - this.ignoreAbove = Integer.MAX_VALUE; - this.nullValue = null; - this.mappedFieldTypeName = null; - } + private final KeywordFieldType valueAndPathFieldType; - public FlatObjectFieldType(String name, FieldType fieldType) { + public FlatObjectFieldType( + String name, + String mappedFieldTypeName, + boolean isSearchable, + boolean hasDocValues, + NamedAnalyzer analyzer, + Map meta + ) { super( name, - fieldType.indexOptions() != IndexOptions.NONE, + isSearchable, false, - true, - new TextSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), - Collections.emptyMap() + hasDocValues, + analyzer == null ? TextSearchInfo.SIMPLE_MATCH_ONLY : new TextSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), + meta ); + setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); this.ignoreAbove = Integer.MAX_VALUE; this.nullValue = null; - this.mappedFieldTypeName = null; - } - - public FlatObjectFieldType(String name, NamedAnalyzer analyzer) { - super(name, true, false, true, new TextSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap()); - this.ignoreAbove = Integer.MAX_VALUE; - this.nullValue = null; - this.mappedFieldTypeName = null; + this.mappedFieldTypeName = mappedFieldTypeName; + this.valueFieldType = getKeywordFieldType(name, VALUE_SUFFIX, isSearchable, hasDocValues); + this.valueAndPathFieldType = getKeywordFieldType(name, VALUE_AND_PATH_SUFFIX, isSearchable, hasDocValues); } - public FlatObjectFieldType(String name, String mappedFieldTypeName) { + public FlatObjectFieldType( + String name, + String mappedFieldTypeName, + KeywordFieldType valueFieldType, + KeywordFieldType valueAndPathFieldType + ) { super( name, - true, + valueFieldType.isSearchable(), false, - true, + valueFieldType.hasDocValues(), new TextSearchInfo(Defaults.FIELD_TYPE, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), Collections.emptyMap() ); this.ignoreAbove = Integer.MAX_VALUE; this.nullValue = null; this.mappedFieldTypeName = mappedFieldTypeName; - } - - void setValueFieldType(KeywordFieldMapper.KeywordFieldType valueFieldType) { this.valueFieldType = valueFieldType; + this.valueAndPathFieldType = valueAndPathFieldType; } - void setValueAndPathFieldType(KeywordFieldMapper.KeywordFieldType ValueAndPathFieldType) { - this.valueAndPathFieldType = ValueAndPathFieldType; + static KeywordFieldType getKeywordFieldType(String fullName, String valueType, boolean isSearchable, boolean hasDocValue) { + return new KeywordFieldType(fullName + valueType, isSearchable, hasDocValue, Collections.emptyMap()) { + @Override + protected String rewriteForDocValue(Object value) { + assert value instanceof String; + return fullName + DOT_SYMBOL + value; + } + }; } - public KeywordFieldMapper.KeywordFieldType getValueFieldType() { + public KeywordFieldType getValueFieldType() { return this.valueFieldType; } - public KeywordFieldMapper.KeywordFieldType getValueAndPathFieldType() { + public KeywordFieldType getValueAndPathFieldType() { return this.valueAndPathFieldType; } @@ -331,6 +332,10 @@ protected BytesRef indexedValueForSearch(Object value) { return getTextSearchInfo().getSearchAnalyzer().normalize(name(), value.toString()); } + private KeywordFieldType valueFieldType() { + return (mappedFieldTypeName == null) ? valueFieldType : valueAndPathFieldType; + } + /** * redirect queries with rewrite value to rewriteSearchValue and directSubFieldName */ @@ -352,17 +357,12 @@ public Query termQuery(Object value, @Nullable QueryShardContext context) { @Override public Query termsQuery(List values, QueryShardContext context) { - failIfNotIndexed(); - String directedSearchFieldName = directSubfield(); - BytesRef[] bytesRefs = new BytesRef[values.size()]; - for (int i = 0; i < bytesRefs.length; i++) { - String rewriteValues = rewriteValue(inputToString(values.get(i))); - - bytesRefs[i] = indexedValueForSearch(new BytesRef(rewriteValues)); - + List parsedValues = new ArrayList<>(values.size()); + for (Object value : values) { + parsedValues.add(rewriteValue(inputToString(value))); } - return new TermInSetQuery(directedSearchFieldName, bytesRefs); + return valueFieldType().termsQuery(parsedValues, context); } /** @@ -395,7 +395,7 @@ public String rewriteValue(String searchValueString) { } - private boolean hasMappedFieldTyeNameInQueryFieldName(String input) { + boolean hasMappedFieldTyeNameInQueryFieldName(String input) { String prefix = this.mappedFieldTypeName; if (prefix == null) { return false; @@ -413,6 +413,9 @@ private boolean hasMappedFieldTyeNameInQueryFieldName(String input) { } private String inputToString(Object inputValue) { + if (inputValue == null) { + return null; + } if (inputValue instanceof Integer) { String inputToString = Integer.toString((Integer) inputValue); return inputToString; @@ -448,46 +451,50 @@ private String inputToString(Object inputValue) { @Override public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, boolean caseInsensitive, QueryShardContext context) { - String directSubfield = directSubfield(); - String rewriteValue = rewriteValue(value); - - if (context.allowExpensiveQueries() == false) { - throw new OpenSearchException( - "[prefix] queries cannot be executed when '" - + ALLOW_EXPENSIVE_QUERIES.getKey() - + "' is set to false. For optimised prefix queries on text " - + "fields please enable [index_prefixes]." - ); - } - failIfNotIndexed(); - if (method == null) { - method = MultiTermQuery.CONSTANT_SCORE_REWRITE; - } - if (caseInsensitive) { - return AutomatonQueries.caseInsensitivePrefixQuery((new Term(directSubfield, indexedValueForSearch(rewriteValue))), method); - } - return new PrefixQuery(new Term(directSubfield, indexedValueForSearch(rewriteValue)), method); + return valueFieldType().prefixQuery(rewriteValue(value), method, caseInsensitive, context); + } + + @Override + public Query regexpQuery( + String value, + int syntaxFlags, + int matchFlags, + int maxDeterminizedStates, + @Nullable MultiTermQuery.RewriteMethod method, + QueryShardContext context + ) { + return valueFieldType().regexpQuery(rewriteValue(value), syntaxFlags, matchFlags, maxDeterminizedStates, method, context); + } + + @Override + public Query fuzzyQuery( + Object value, + Fuzziness fuzziness, + int prefixLength, + int maxExpansions, + boolean transpositions, + @Nullable MultiTermQuery.RewriteMethod method, + QueryShardContext context + ) { + return valueFieldType().fuzzyQuery( + rewriteValue(inputToString(value)), + fuzziness, + prefixLength, + maxExpansions, + transpositions, + method, + context + ); } @Override public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) { - String directSubfield = directSubfield(); - String rewriteUpperTerm = rewriteValue(inputToString(upperTerm)); - String rewriteLowerTerm = rewriteValue(inputToString(lowerTerm)); - if (context.allowExpensiveQueries() == false) { - throw new OpenSearchException( - "[range] queries on [text] or [keyword] fields cannot be executed when '" - + ALLOW_EXPENSIVE_QUERIES.getKey() - + "' is set to false." - ); - } - failIfNotIndexed(); - return new TermRangeQuery( - directSubfield, - lowerTerm == null ? null : indexedValueForSearch(rewriteLowerTerm), - upperTerm == null ? null : indexedValueForSearch(rewriteUpperTerm), + return valueFieldType().rangeQuery( + rewriteValue(inputToString(lowerTerm)), + rewriteValue(inputToString(upperTerm)), includeLower, - includeUpper + includeUpper, + context ); } @@ -503,8 +510,12 @@ public Query existsQuery(QueryShardContext context) { searchKey = this.mappedFieldTypeName; searchField = name(); } else { - searchKey = FieldNamesFieldMapper.NAME; - searchField = name(); + if (hasDocValues()) { + return new FieldExistsQuery(name()); + } else { + searchKey = FieldNamesFieldMapper.NAME; + searchField = name(); + } } return new TermQuery(new Term(searchKey, indexedValueForSearch(searchField))); } @@ -516,13 +527,7 @@ public Query wildcardQuery( boolean caseInsensitve, QueryShardContext context ) { - // flat_object field types are always normalized, so ignore case sensitivity and force normalize the wildcard - // query text - throw new QueryShardException( - context, - "Can only use wildcard queries on keyword and text fields - not on [" + name() + "] which is of type [" + typeName() + "]" - ); - + return valueFieldType().wildcardQuery(rewriteValue(value), method, caseInsensitve, context); } } @@ -606,7 +611,6 @@ protected void parseCreateField(ParseContext context) throws IOException { } } } - } @Override @@ -637,6 +641,8 @@ public Iterator iterator() { */ private void parseValueAddFields(ParseContext context, String value, String fieldName) throws IOException { + assert valueFieldMapper != null; + assert valueAndPathFieldMapper != null; NamedAnalyzer normalizer = fieldType().normalizer(); if (normalizer != null) { value = normalizeValue(normalizer, name(), value); @@ -647,69 +653,57 @@ private void parseValueAddFields(ParseContext context, String value, String fiel if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { // convert to utf8 only once before feeding postings/dv/stored fields - final BytesRef binaryValue = new BytesRef(fieldType().name() + DOT_SYMBOL + value); - Field field = new FlatObjectField(fieldType().name(), binaryValue, fieldType); - if (fieldType().hasDocValues() == false && fieldType.omitNorms()) { + if (fieldType().hasDocValues() == false) { createFieldNamesField(context); } if (fieldName.equals(fieldType().name())) { + Field field = new FlatObjectField(fieldType().name(), binaryValue, fieldType); context.doc().add(field); - } - if (valueType.equals(VALUE_SUFFIX)) { - if (valueFieldMapper != null) { - valueFieldMapper.addField(context, value); - } - } - if (valueType.equals(VALUE_AND_PATH_SUFFIX)) { - if (valueAndPathFieldMapper != null) { - valueAndPathFieldMapper.addField(context, value); - } + } else if (valueType.equals(VALUE_SUFFIX)) { + valueFieldMapper.addField(context, value); + } else if (valueType.equals(VALUE_AND_PATH_SUFFIX)) { + valueAndPathFieldMapper.addField(context, value); } if (fieldType().hasDocValues()) { if (fieldName.equals(fieldType().name())) { context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue)); - } - if (valueType.equals(VALUE_SUFFIX)) { - if (valueFieldMapper != null) { - context.doc().add(new SortedSetDocValuesField(fieldType().name() + VALUE_SUFFIX, binaryValue)); - } - } - if (valueType.equals(VALUE_AND_PATH_SUFFIX)) { - if (valueAndPathFieldMapper != null) { - context.doc().add(new SortedSetDocValuesField(fieldType().name() + VALUE_AND_PATH_SUFFIX, binaryValue)); - } + } else if (valueType.equals(VALUE_SUFFIX)) { + context.doc().add(new SortedSetDocValuesField(fieldType().name() + VALUE_SUFFIX, binaryValue)); + } else if (valueType.equals(VALUE_AND_PATH_SUFFIX)) { + context.doc().add(new SortedSetDocValuesField(fieldType().name() + VALUE_AND_PATH_SUFFIX, binaryValue)); } } - } - } private static String normalizeValue(NamedAnalyzer normalizer, String field, String value) throws IOException { - String normalizerErrorMessage = "The normalization token stream is " - + "expected to produce exactly 1 token, but got 0 for analyzer " - + normalizer - + " and input \"" - + value - + "\""; try (TokenStream ts = normalizer.tokenStream(field, value)) { final CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); ts.reset(); if (ts.incrementToken() == false) { - throw new IllegalStateException(normalizerErrorMessage); + throw new IllegalStateException(errorMessage(normalizer, value)); } final String newValue = termAtt.toString(); if (ts.incrementToken()) { - throw new IllegalStateException(normalizerErrorMessage); + throw new IllegalStateException(errorMessage(normalizer, value)); } ts.end(); return newValue; } } + private static String errorMessage(NamedAnalyzer normalizer, String value) { + return "The normalization token stream is " + + "expected to produce exactly 1 token, but got 0 for analyzer " + + normalizer + + " and input \"" + + value + + "\""; + } + @Override protected String contentType() { return CONTENT_TYPE; @@ -717,7 +711,7 @@ protected String contentType() { private static final class ValueAndPathFieldMapper extends FieldMapper { - protected ValueAndPathFieldMapper(FieldType fieldType, KeywordFieldMapper.KeywordFieldType mappedFieldType) { + protected ValueAndPathFieldMapper(FieldType fieldType, KeywordFieldType mappedFieldType) { super(mappedFieldType.name(), fieldType, mappedFieldType, MultiFields.empty(), CopyTo.empty()); } @@ -728,7 +722,7 @@ void addField(ParseContext context, String value) { context.doc().add(field); - if (fieldType().hasDocValues() == false && fieldType.omitNorms()) { + if (fieldType().hasDocValues() == false) { createFieldNamesField(context); } } @@ -758,7 +752,7 @@ public String toString() { private static final class ValueFieldMapper extends FieldMapper { - protected ValueFieldMapper(FieldType fieldType, KeywordFieldMapper.KeywordFieldType mappedFieldType) { + protected ValueFieldMapper(FieldType fieldType, KeywordFieldType mappedFieldType) { super(mappedFieldType.name(), fieldType, mappedFieldType, MultiFields.empty(), CopyTo.empty()); } @@ -768,7 +762,7 @@ void addField(ParseContext context, String value) { Field field = new KeywordFieldMapper.KeywordField(fieldType().name(), binaryValue, fieldType); context.doc().add(field); - if (fieldType().hasDocValues() == false && fieldType.omitNorms()) { + if (fieldType().hasDocValues() == false) { createFieldNamesField(context); } } diff --git a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java index 11ff601b3fd6d..54a1aead5fcc7 100644 --- a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java @@ -263,7 +263,7 @@ public KeywordFieldMapper build(BuilderContext context) { * * @opensearch.internal */ - public static final class KeywordFieldType extends StringFieldType { + public static class KeywordFieldType extends StringFieldType { private final int ignoreAbove; private final String nullValue; @@ -387,6 +387,10 @@ protected BytesRef indexedValueForSearch(Object value) { return getTextSearchInfo().getSearchAnalyzer().normalize(name(), value.toString()); } + protected Object rewriteForDocValue(Object value) { + return value; + } + @Override public Query termsQuery(List values, QueryShardContext context) { failIfNotIndexedAndNoDocValues(); @@ -395,19 +399,21 @@ public Query termsQuery(List values, QueryShardContext context) { if (!context.keywordFieldIndexOrDocValuesEnabled()) { return super.termsQuery(values, context); } - BytesRef[] bytesRefs = new BytesRef[values.size()]; - for (int i = 0; i < bytesRefs.length; i++) { - bytesRefs[i] = indexedValueForSearch(values.get(i)); + BytesRef[] iBytesRefs = new BytesRef[values.size()]; + BytesRef[] dVByteRefs = new BytesRef[values.size()]; + for (int i = 0; i < iBytesRefs.length; i++) { + iBytesRefs[i] = indexedValueForSearch(values.get(i)); + dVByteRefs[i] = indexedValueForSearch(rewriteForDocValue(values.get(i))); } - Query indexQuery = new TermInSetQuery(name(), bytesRefs); - Query dvQuery = new TermInSetQuery(MultiTermQuery.DOC_VALUES_REWRITE, name(), bytesRefs); + Query indexQuery = new TermInSetQuery(name(), iBytesRefs); + Query dvQuery = new TermInSetQuery(MultiTermQuery.DOC_VALUES_REWRITE, name(), dVByteRefs); return new IndexOrDocValuesQuery(indexQuery, dvQuery); } // if we only have doc_values enabled, we construct a new query with doc_values re-written if (hasDocValues()) { BytesRef[] bytesRefs = new BytesRef[values.size()]; for (int i = 0; i < bytesRefs.length; i++) { - bytesRefs[i] = indexedValueForSearch(values.get(i)); + bytesRefs[i] = indexedValueForSearch(rewriteForDocValue(values.get(i))); } return new TermInSetQuery(MultiTermQuery.DOC_VALUES_REWRITE, name(), bytesRefs); } @@ -436,17 +442,25 @@ public Query prefixQuery( return super.prefixQuery(value, method, caseInsensitive, context); } Query indexQuery = super.prefixQuery(value, method, caseInsensitive, context); - Query dvQuery = super.prefixQuery(value, MultiTermQuery.DOC_VALUES_REWRITE, caseInsensitive, context); + Query dvQuery = super.prefixQuery( + (String) rewriteForDocValue(value), + MultiTermQuery.DOC_VALUES_REWRITE, + caseInsensitive, + context + ); return new IndexOrDocValuesQuery(indexQuery, dvQuery); } if (hasDocValues()) { if (caseInsensitive) { return AutomatonQueries.caseInsensitivePrefixQuery( - (new Term(name(), indexedValueForSearch(value))), + (new Term(name(), indexedValueForSearch(rewriteForDocValue(value)))), MultiTermQuery.DOC_VALUES_REWRITE ); } - return new PrefixQuery(new Term(name(), indexedValueForSearch(value)), MultiTermQuery.DOC_VALUES_REWRITE); + return new PrefixQuery( + new Term(name(), indexedValueForSearch(rewriteForDocValue(value))), + MultiTermQuery.DOC_VALUES_REWRITE + ); } return super.prefixQuery(value, method, caseInsensitive, context); } @@ -472,7 +486,7 @@ public Query regexpQuery( } Query indexQuery = super.regexpQuery(value, syntaxFlags, matchFlags, maxDeterminizedStates, method, context); Query dvQuery = super.regexpQuery( - value, + (String) rewriteForDocValue(value), syntaxFlags, matchFlags, maxDeterminizedStates, @@ -483,7 +497,7 @@ public Query regexpQuery( } if (hasDocValues()) { return new RegexpQuery( - new Term(name(), indexedValueForSearch(value)), + new Term(name(), indexedValueForSearch(rewriteForDocValue(value))), syntaxFlags, matchFlags, RegexpQuery.DEFAULT_PROVIDER, @@ -514,8 +528,8 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower ); Query dvQuery = new TermRangeQuery( name(), - lowerTerm == null ? null : indexedValueForSearch(lowerTerm), - upperTerm == null ? null : indexedValueForSearch(upperTerm), + lowerTerm == null ? null : indexedValueForSearch(rewriteForDocValue(lowerTerm)), + upperTerm == null ? null : indexedValueForSearch(rewriteForDocValue(upperTerm)), includeLower, includeUpper, MultiTermQuery.DOC_VALUES_REWRITE @@ -525,8 +539,8 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower if (hasDocValues()) { return new TermRangeQuery( name(), - lowerTerm == null ? null : indexedValueForSearch(lowerTerm), - upperTerm == null ? null : indexedValueForSearch(upperTerm), + lowerTerm == null ? null : indexedValueForSearch(rewriteForDocValue(lowerTerm)), + upperTerm == null ? null : indexedValueForSearch(rewriteForDocValue(upperTerm)), includeLower, includeUpper, MultiTermQuery.DOC_VALUES_REWRITE @@ -563,7 +577,7 @@ public Query fuzzyQuery( } Query indexQuery = super.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, method, context); Query dvQuery = super.fuzzyQuery( - value, + rewriteForDocValue(value), fuzziness, prefixLength, maxExpansions, @@ -575,8 +589,8 @@ public Query fuzzyQuery( } if (hasDocValues()) { return new FuzzyQuery( - new Term(name(), indexedValueForSearch(value)), - fuzziness.asDistance(BytesRefs.toString(value)), + new Term(name(), indexedValueForSearch(rewriteForDocValue(value))), + fuzziness.asDistance(BytesRefs.toString(rewriteForDocValue(value))), prefixLength, maxExpansions, transpositions, @@ -607,13 +621,19 @@ public Query wildcardQuery( return super.wildcardQuery(value, method, caseInsensitive, true, context); } Query indexQuery = super.wildcardQuery(value, method, caseInsensitive, true, context); - Query dvQuery = super.wildcardQuery(value, MultiTermQuery.DOC_VALUES_REWRITE, caseInsensitive, true, context); + Query dvQuery = super.wildcardQuery( + (String) rewriteForDocValue(value), + MultiTermQuery.DOC_VALUES_REWRITE, + caseInsensitive, + true, + context + ); return new IndexOrDocValuesQuery(indexQuery, dvQuery); } if (hasDocValues()) { Term term; value = normalizeWildcardPattern(name(), value, getTextSearchInfo().getSearchAnalyzer()); - term = new Term(name(), value); + term = new Term(name(), (String) rewriteForDocValue(value)); if (caseInsensitive) { return AutomatonQueries.caseInsensitiveWildcardQuery(term, method); } diff --git a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java index 94d1f501bee51..afd9e994ce3ae 100644 --- a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java @@ -12,6 +12,7 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; +import org.apache.lucene.search.FieldExistsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; @@ -23,14 +24,13 @@ import java.io.IOException; +import static org.opensearch.common.xcontent.JsonToStringXContentParser.VALUE_AND_PATH_SUFFIX; +import static org.opensearch.common.xcontent.JsonToStringXContentParser.VALUE_SUFFIX; +import static org.opensearch.index.mapper.FlatObjectFieldMapper.CONTENT_TYPE; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.core.IsEqual.equalTo; public class FlatObjectFieldMapperTests extends MapperTestCase { - private static final String FIELD_TYPE = "flat_object"; - private static final String VALUE_AND_PATH_SUFFIX = "._valueAndPath"; - private static final String VALUE_SUFFIX = "._value"; - protected boolean supportsMeta() { return false; } @@ -41,7 +41,7 @@ protected boolean supportsOrIgnoresBoost() { public void testMapperServiceHasParser() throws IOException { MapperService mapperService = createMapperService(fieldMapping(b -> { minimalMapping(b); })); - Mapper.TypeParser parser = mapperService.mapperRegistry.getMapperParsers().get(FIELD_TYPE); + Mapper.TypeParser parser = mapperService.mapperRegistry.getMapperParsers().get(CONTENT_TYPE); assertNotNull(parser); assertTrue(parser instanceof FlatObjectFieldMapper.TypeParser); } @@ -49,28 +49,39 @@ public void testMapperServiceHasParser() throws IOException { protected void assertExistsQuery(MapperService mapperService) throws IOException { ParseContext.Document fields = mapperService.documentMapper().parse(source(this::writeField)).rootDoc(); QueryShardContext queryShardContext = createQueryShardContext(mapperService); - MappedFieldType fieldType = mapperService.fieldType("field"); + FlatObjectFieldMapper.FlatObjectFieldType fieldType = (FlatObjectFieldMapper.FlatObjectFieldType) mapperService.fieldType("field"); Query query = fieldType.existsQuery(queryShardContext); assertExistsQuery(fieldType, query, fields); - } - protected void assertExistsQuery(MappedFieldType fieldType, Query query, ParseContext.Document fields) { - // we always perform a term query against _field_names, even when the field - // is not added to _field_names because it is not indexed nor stored - assertThat(query, instanceOf(TermQuery.class)); - TermQuery termQuery = (TermQuery) query; - assertEquals(FieldNamesFieldMapper.NAME, termQuery.getTerm().field()); - assertEquals("field", termQuery.getTerm().text()); - if (fieldType.isSearchable() || fieldType.isStored()) { - assertNotNull(fields.getField(FieldNamesFieldMapper.NAME)); + protected void assertExistsQuery(FlatObjectFieldMapper.FlatObjectFieldType fieldType, Query query, ParseContext.Document fields) { + + if (fieldType.hasDocValues() && fieldType.hasMappedFieldTyeNameInQueryFieldName(fieldType.name()) == false) { + assertThat(query, instanceOf(FieldExistsQuery.class)); + FieldExistsQuery fieldExistsQuery = (FieldExistsQuery) query; + assertEquals(fieldType.name(), fieldExistsQuery.getField()); } else { + assertThat(query, instanceOf(TermQuery.class)); + TermQuery termQuery = (TermQuery) query; + assertEquals(FieldNamesFieldMapper.NAME, termQuery.getTerm().field()); + assertEquals("field", termQuery.getTerm().text()); + } + + if (fieldType.hasDocValues()) { + assertDocValuesField(fields, "field"); assertNoFieldNamesField(fields); + } else { + assertNoDocValuesField(fields, "field"); + if (fieldType.isSearchable()) { + assertNotNull(fields.getField(FieldNamesFieldMapper.NAME)); + } else { + assertNoFieldNamesField(fields); + } } } public void minimalMapping(XContentBuilder b) throws IOException { - b.field("type", FIELD_TYPE); + b.field("type", CONTENT_TYPE); } /** diff --git a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldTypeTests.java index 9ec053dc59d10..38a6f13777f00 100644 --- a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldTypeTests.java @@ -8,11 +8,23 @@ package org.opensearch.index.mapper; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; +import org.apache.lucene.search.FieldExistsQuery; +import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.IndexOrDocValuesQuery; +import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.PrefixQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.RegexpQuery; +import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.search.TermQuery; -import org.opensearch.Version; -import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.settings.Settings; +import org.apache.lucene.search.TermRangeQuery; +import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.Operations; +import org.opensearch.common.unit.Fuzziness; import org.opensearch.index.analysis.AnalyzerScope; import org.opensearch.index.analysis.NamedAnalyzer; @@ -21,18 +33,41 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; +import static org.opensearch.common.xcontent.JsonToStringXContentParser.VALUE_AND_PATH_SUFFIX; +import static org.opensearch.common.xcontent.JsonToStringXContentParser.VALUE_SUFFIX; +import static org.apache.lucene.search.MultiTermQuery.CONSTANT_SCORE_REWRITE; +import static org.apache.lucene.search.MultiTermQuery.DOC_VALUES_REWRITE; + public class FlatObjectFieldTypeTests extends FieldTypeTestCase { - private static MappedFieldType getFlatParentFieldType(String fieldName) { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - MappedFieldType flatParentFieldType = new FlatObjectFieldMapper.Builder(fieldName).build(context).fieldType(); - return flatParentFieldType; + + private static MappedFieldType getFlatParentFieldType( + String fieldName, + String mappedFieldTypeName, + boolean isSearchable, + boolean hasDocValues + ) { + FlatObjectFieldMapper.Builder builder = new FlatObjectFieldMapper.Builder(fieldName); + FlatObjectFieldMapper.FlatObjectFieldType flatObjectFieldType = new FlatObjectFieldMapper.FlatObjectFieldType( + fieldName, + mappedFieldTypeName, + isSearchable, + hasDocValues, + null, + Collections.emptyMap() + ); + FieldType fieldtype = new FieldType(FlatObjectFieldMapper.Defaults.FIELD_TYPE); + FieldType vft = new FieldType(fieldtype); + if (flatObjectFieldType.isSearchable() == false) { + vft.setIndexOptions(IndexOptions.NONE); + } + return flatObjectFieldType; } public void testFetchSourceValue() throws IOException { - MappedFieldType mapper = getFlatParentFieldType("field"); + MappedFieldType mapper = getFlatParentFieldType("field", null, true, true); Map jsonPoint = new HashMap<>(); jsonPoint.put("type", "flat_object"); @@ -54,32 +89,48 @@ public void testFetchSourceValue() throws IOException { public void testDirectSubfield() { { - MappedFieldType flatParentFieldType = getFlatParentFieldType("field"); + FlatObjectFieldMapper.FlatObjectFieldType flatParentFieldType = + (FlatObjectFieldMapper.FlatObjectFieldType) (getFlatParentFieldType("field", null, true, true)); // when searching for "foo" in "field", the directSubfield is field._value field - String searchFieldName = ((FlatObjectFieldMapper.FlatObjectFieldType) flatParentFieldType).directSubfield(); + String searchFieldName = (flatParentFieldType).directSubfield(); assertEquals("field._value", searchFieldName); - MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType("bar", flatParentFieldType.name()); + MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType( + "bar", + flatParentFieldType.name(), + flatParentFieldType.getValueFieldType(), + flatParentFieldType.getValueAndPathFieldType() + ); // when searching for "foo" in "field.bar", the directSubfield is field._valueAndPath field String searchFieldNameDocPath = ((FlatObjectFieldMapper.FlatObjectFieldType) dynamicMappedFieldType).directSubfield(); assertEquals("field._valueAndPath", searchFieldNameDocPath); } { NamedAnalyzer analyzer = new NamedAnalyzer("default", AnalyzerScope.INDEX, null); - MappedFieldType ft = new FlatObjectFieldMapper.FlatObjectFieldType("field", analyzer); + MappedFieldType ft = new FlatObjectFieldMapper.FlatObjectFieldType("field", null, true, true, analyzer, Collections.emptyMap()); assertEquals("field._value", ((FlatObjectFieldMapper.FlatObjectFieldType) ft).directSubfield()); } } public void testRewriteValue() { - MappedFieldType flatParentFieldType = getFlatParentFieldType("field"); + FlatObjectFieldMapper.FlatObjectFieldType flatParentFieldType = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + true + ); // when searching for "foo" in "field", the rewrite value is "foo" - String searchValues = ((FlatObjectFieldMapper.FlatObjectFieldType) flatParentFieldType).rewriteValue("foo"); + String searchValues = (flatParentFieldType).rewriteValue("foo"); assertEquals("foo", searchValues); - MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType("field.bar", flatParentFieldType.name()); + MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType( + "field.bar", + flatParentFieldType.name(), + flatParentFieldType.getValueFieldType(), + flatParentFieldType.getValueAndPathFieldType() + ); // when searching for "foo" in "field.bar", the rewrite value is "field.bar=foo" String searchFieldNameDocPath = ((FlatObjectFieldMapper.FlatObjectFieldType) dynamicMappedFieldType).directSubfield(); @@ -89,15 +140,25 @@ public void testRewriteValue() { public void testTermQuery() { - MappedFieldType flatParentFieldType = getFlatParentFieldType("field"); + FlatObjectFieldMapper.FlatObjectFieldType flatParentFieldType = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + true + ); // when searching for "foo" in "field", the term query is directed to search "foo" in field._value field - String searchFieldName = ((FlatObjectFieldMapper.FlatObjectFieldType) flatParentFieldType).directSubfield(); - String searchValues = ((FlatObjectFieldMapper.FlatObjectFieldType) flatParentFieldType).rewriteValue("foo"); + String searchFieldName = (flatParentFieldType).directSubfield(); + String searchValues = (flatParentFieldType).rewriteValue("foo"); assertEquals("foo", searchValues); assertEquals(new TermQuery(new Term(searchFieldName, searchValues)), flatParentFieldType.termQuery(searchValues, null)); - MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType("field.bar", flatParentFieldType.name()); + MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType( + "field.bar", + flatParentFieldType.name(), + flatParentFieldType.getValueFieldType(), + flatParentFieldType.getValueAndPathFieldType() + ); // when searching for "foo" in "field.bar", the term query is directed to search in field._valueAndPath field String searchFieldNameDocPath = ((FlatObjectFieldMapper.FlatObjectFieldType) dynamicMappedFieldType).directSubfield(); @@ -105,30 +166,919 @@ public void testTermQuery() { assertEquals("field.bar=foo", searchValuesDocPath); assertEquals(new TermQuery(new Term(searchFieldNameDocPath, searchValuesDocPath)), dynamicMappedFieldType.termQuery("foo", null)); - MappedFieldType unsearchable = new FlatObjectFieldMapper.FlatObjectFieldType("field", false, true, Collections.emptyMap()); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("bar", null)); + MappedFieldType unsearchable = new FlatObjectFieldMapper.FlatObjectFieldType( + "field", + null, + false, + true, + null, + Collections.emptyMap() + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> unsearchable.termQuery("bar", MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); } public void testExistsQuery() { { - MappedFieldType ft = getFlatParentFieldType("field"); + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + true + ); // when checking on the flat_object field name "field", check if exist in the field mapper names - assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, "field")), ft.existsQuery(null)); + assertEquals(new FieldExistsQuery("field"), ft.existsQuery(null)); // when checking if a subfield within the flat_object, for example, "field.bar", use term query in the flat_object field - MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType("field.bar", ft.name()); + MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType( + "field.bar", + ft.name(), + ft.getValueFieldType(), + ft.getValueAndPathFieldType() + ); assertEquals(new TermQuery(new Term("field", "field.bar")), dynamicMappedFieldType.existsQuery(null)); } + { FlatObjectFieldMapper.FlatObjectFieldType ft = new FlatObjectFieldMapper.FlatObjectFieldType( "field", + null, true, false, + null, Collections.emptyMap() ); - assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, "field")), ft.existsQuery(null)); + assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, "field")), ft.existsQuery(MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + } + + public void testTermsQuery() { + + // 1.test isSearchable=true, hasDocValues=true, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + true + ); + List indexTerms = new ArrayList<>(); + indexTerms.add(new BytesRef("foo")); + indexTerms.add(new BytesRef("bar")); + List docValueterms = new ArrayList<>(); + docValueterms.add(new BytesRef("field.foo")); + docValueterms.add(new BytesRef("field.bar")); + Query expected = new IndexOrDocValuesQuery( + new TermInSetQuery("field" + VALUE_SUFFIX, indexTerms), + new TermInSetQuery(DOC_VALUES_REWRITE, "field" + VALUE_SUFFIX, docValueterms) + ); + + assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=true, hasDocValues=true, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + true, + true + ); + + List indexTerms = new ArrayList<>(); + indexTerms.add(new BytesRef("foo")); + indexTerms.add(new BytesRef("bar")); + List docValueterms = new ArrayList<>(); + docValueterms.add(new BytesRef("field.foo")); + docValueterms.add(new BytesRef("field.bar")); + Query expected = new IndexOrDocValuesQuery( + new TermInSetQuery("field" + VALUE_AND_PATH_SUFFIX, indexTerms), + new TermInSetQuery(DOC_VALUES_REWRITE, "field" + VALUE_AND_PATH_SUFFIX, docValueterms) + ); + + assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 2.test isSearchable=true, hasDocValues=false, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + false + ); + List indexTerms = new ArrayList<>(); + indexTerms.add(new BytesRef("foo")); + indexTerms.add(new BytesRef("bar")); + Query expected = new TermInSetQuery("field" + VALUE_SUFFIX, indexTerms); + + assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + true, + false + ); + List indexTerms = new ArrayList<>(); + indexTerms.add(new BytesRef("foo")); + indexTerms.add(new BytesRef("bar")); + Query expected = new TermInSetQuery("field" + VALUE_AND_PATH_SUFFIX, indexTerms); + + assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), null)); + } + + // 3.test isSearchable=false, hasDocValues=true, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + false, + true + ); + + List indexTerms = new ArrayList<>(); + indexTerms.add(new BytesRef("foo")); + indexTerms.add(new BytesRef("bar")); + List docValueterms = new ArrayList<>(); + docValueterms.add(new BytesRef("field.foo")); + docValueterms.add(new BytesRef("field.bar")); + Query expected = new TermInSetQuery(DOC_VALUES_REWRITE, "field" + VALUE_SUFFIX, docValueterms); + + assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + false, + true + ); + + List indexTerms = new ArrayList<>(); + indexTerms.add(new BytesRef("foo")); + indexTerms.add(new BytesRef("bar")); + List docValueterms = new ArrayList<>(); + docValueterms.add(new BytesRef("field.foo")); + docValueterms.add(new BytesRef("field.bar")); + Query expected = new TermInSetQuery(DOC_VALUES_REWRITE, "field" + VALUE_AND_PATH_SUFFIX, docValueterms); + + assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 4.test isSearchable=false, hasDocValues=false, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + false, + false + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); + } + + // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + false, + false + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); + } + } + + public void testPrefixQuery() { + // 1.test isSearchable=true, hasDocValues=true, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + true + ); + Query expected = new IndexOrDocValuesQuery( + new PrefixQuery(new Term("field" + VALUE_SUFFIX, "foo"), CONSTANT_SCORE_REWRITE), + new PrefixQuery(new Term("field" + VALUE_SUFFIX, "field.foo"), DOC_VALUES_REWRITE) + ); + assertEquals(expected, ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=true, hasDocValues=true, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + true, + true + ); + Query expected = new IndexOrDocValuesQuery( + new PrefixQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "foo"), CONSTANT_SCORE_REWRITE), + new PrefixQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "field.foo"), DOC_VALUES_REWRITE) + ); + assertEquals(expected, ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 2.test isSearchable=true, hasDocValues=false, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + false + ); + Query expected = new PrefixQuery(new Term("field" + VALUE_SUFFIX, "foo"), CONSTANT_SCORE_REWRITE); + assertEquals(expected, ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + true, + false + ); + Query expected = new PrefixQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "foo"), CONSTANT_SCORE_REWRITE); + assertEquals(expected, ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 3.test isSearchable=false, hasDocValues=true, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + false, + true + ); + Query expected = new PrefixQuery(new Term("field" + VALUE_SUFFIX, "field.foo"), DOC_VALUES_REWRITE); + assertEquals(expected, ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + false, + true + ); + Query expected = new PrefixQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "field.foo"), DOC_VALUES_REWRITE); + assertEquals(expected, ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 4.test isSearchable=false, hasDocValues=false, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + false, + false + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); + } + + // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + false, + false + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); + } + } + + public void testRegexpQuery() { + // 1.test isSearchable=true, hasDocValues=true, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + true + ); + Query expected = new IndexOrDocValuesQuery( + new RegexpQuery( + new Term("field" + VALUE_SUFFIX, new BytesRef("foo")), + 0, + 0, + RegexpQuery.DEFAULT_PROVIDER, + 10, + CONSTANT_SCORE_REWRITE + ), + new RegexpQuery( + new Term("field" + VALUE_SUFFIX, new BytesRef("field.foo")), + 0, + 0, + RegexpQuery.DEFAULT_PROVIDER, + 10, + DOC_VALUES_REWRITE + ) + ); + assertEquals(expected, ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=true, hasDocValues=true, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + true, + true + ); + Query expected = new IndexOrDocValuesQuery( + new RegexpQuery( + new Term("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("foo")), + 0, + 0, + RegexpQuery.DEFAULT_PROVIDER, + 10, + CONSTANT_SCORE_REWRITE + ), + new RegexpQuery( + new Term("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("field.foo")), + 0, + 0, + RegexpQuery.DEFAULT_PROVIDER, + 10, + DOC_VALUES_REWRITE + ) + ); + assertEquals(expected, ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 2.test isSearchable=true, hasDocValues=false, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + false + ); + Query expected = new RegexpQuery( + new Term("field" + VALUE_SUFFIX, new BytesRef("foo")), + 0, + 0, + RegexpQuery.DEFAULT_PROVIDER, + 10, + CONSTANT_SCORE_REWRITE + ); + assertEquals(expected, ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + true, + false + ); + Query expected = new RegexpQuery( + new Term("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("foo")), + 0, + 0, + RegexpQuery.DEFAULT_PROVIDER, + 10, + CONSTANT_SCORE_REWRITE + ); + assertEquals(expected, ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 3.test isSearchable=false, hasDocValues=true, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + false, + true + ); + Query expected = new RegexpQuery( + new Term("field" + VALUE_SUFFIX, new BytesRef("field.foo")), + 0, + 0, + RegexpQuery.DEFAULT_PROVIDER, + 10, + DOC_VALUES_REWRITE + ); + assertEquals(expected, ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + false, + true + ); + Query expected = new RegexpQuery( + new Term("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("field.foo")), + 0, + 0, + RegexpQuery.DEFAULT_PROVIDER, + 10, + DOC_VALUES_REWRITE + ); + assertEquals(expected, ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 4.test isSearchable=false, hasDocValues=false, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + false, + false + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); + } + + // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + false, + false + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); + } + } + + public void testFuzzyQuery() { + // 1.test isSearchable=true, hasDocValues=true, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + true + ); + Query expected = new IndexOrDocValuesQuery( + new FuzzyQuery(new Term("field" + VALUE_SUFFIX, "foo"), 2, 1, 50, true), + new FuzzyQuery(new Term("field" + VALUE_SUFFIX, "field.foo"), 2, 1, 50, true, MultiTermQuery.DOC_VALUES_REWRITE) + ); + assertEquals(expected, ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=true, hasDocValues=true, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + true, + true + ); + Query expected = new IndexOrDocValuesQuery( + new FuzzyQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "foo"), 2, 1, 50, true), + new FuzzyQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "field.foo"), 2, 1, 50, true, MultiTermQuery.DOC_VALUES_REWRITE) + ); + assertEquals(expected, ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 2.test isSearchable=true, hasDocValues=false, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + false + ); + Query expected = new FuzzyQuery(new Term("field" + VALUE_SUFFIX, "foo"), 2, 1, 50, true); + assertEquals(expected, ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + true, + false + ); + Query expected = new FuzzyQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "foo"), 2, 1, 50, true); + assertEquals(expected, ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 3.test isSearchable=false, hasDocValues=true, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + false, + true + ); + Query expected = new FuzzyQuery( + new Term("field" + VALUE_SUFFIX, "field.foo"), + 2, + 1, + 50, + true, + MultiTermQuery.DOC_VALUES_REWRITE + ); + assertEquals(expected, ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + false, + true + ); + Query expected = new FuzzyQuery( + new Term("field" + VALUE_AND_PATH_SUFFIX, "field.foo"), + 2, + 1, + 50, + true, + MultiTermQuery.DOC_VALUES_REWRITE + ); + assertEquals(expected, ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 4.test isSearchable=false, hasDocValues=false, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + false, + false + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); + } + + // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + false, + false + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); + } + } + + public void testRangeQuery() { + // 1.test isSearchable=true, hasDocValues=true, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + true + ); + Query expected = new IndexOrDocValuesQuery( + new TermRangeQuery("field" + VALUE_SUFFIX, new BytesRef("2"), new BytesRef("10"), true, true), + new TermRangeQuery( + "field" + VALUE_SUFFIX, + new BytesRef("field.2"), + new BytesRef("field.10"), + true, + true, + DOC_VALUES_REWRITE + ) + ); + assertEquals(expected, ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=true, hasDocValues=true, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + true, + true + ); + Query expected = new IndexOrDocValuesQuery( + new TermRangeQuery("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("2"), new BytesRef("10"), true, true), + new TermRangeQuery( + "field" + VALUE_AND_PATH_SUFFIX, + new BytesRef("field.2"), + new BytesRef("field.10"), + true, + true, + DOC_VALUES_REWRITE + ) + ); + assertEquals(expected, ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 2.test isSearchable=true, hasDocValues=false, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + false + ); + Query expected = new TermRangeQuery("field" + VALUE_SUFFIX, new BytesRef("2"), new BytesRef("10"), true, true); + assertEquals(expected, ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + true, + false + ); + Query expected = new TermRangeQuery("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("2"), new BytesRef("10"), true, true); + assertEquals(expected, ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 3.test isSearchable=false, hasDocValues=true, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + false, + true + ); + Query expected = new TermRangeQuery( + "field" + VALUE_SUFFIX, + new BytesRef("field.2"), + new BytesRef("field.10"), + true, + true, + DOC_VALUES_REWRITE + ); + assertEquals(expected, ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + false, + true + ); + Query expected = new TermRangeQuery( + "field" + VALUE_AND_PATH_SUFFIX, + new BytesRef("field.2"), + new BytesRef("field.10"), + true, + true, + DOC_VALUES_REWRITE + ); + assertEquals(expected, ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 4.test isSearchable=false, hasDocValues=false, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + false, + false + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); + } + + // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + false, + false + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); + } + } + + public void testWildcardQuery() { + // 1.test isSearchable=true, hasDocValues=true, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + true + ); + Query expected = new IndexOrDocValuesQuery( + new WildcardQuery( + new Term("field" + VALUE_SUFFIX, "foo*"), + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + MultiTermQuery.CONSTANT_SCORE_REWRITE + ), + new WildcardQuery( + new Term("field" + VALUE_SUFFIX, "field.foo*"), + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + MultiTermQuery.DOC_VALUES_REWRITE + ) + ); + assertEquals(expected, ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=true, hasDocValues=true, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + true, + true + ); + Query expected = new IndexOrDocValuesQuery( + new WildcardQuery( + new Term("field" + VALUE_AND_PATH_SUFFIX, "foo*"), + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + MultiTermQuery.CONSTANT_SCORE_REWRITE + ), + new WildcardQuery( + new Term("field" + VALUE_AND_PATH_SUFFIX, "field.foo*"), + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + MultiTermQuery.DOC_VALUES_REWRITE + ) + ); + assertEquals(expected, ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 2.test isSearchable=true, hasDocValues=false, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + false + ); + Query expected = new WildcardQuery( + new Term("field" + VALUE_SUFFIX, "foo*"), + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + MultiTermQuery.CONSTANT_SCORE_REWRITE + ); + assertEquals(expected, ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + true, + false + ); + Query expected = new WildcardQuery( + new Term("field" + VALUE_AND_PATH_SUFFIX, "foo*"), + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + MultiTermQuery.CONSTANT_SCORE_REWRITE + ); + assertEquals(expected, ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 3.test isSearchable=false, hasDocValues=true, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + false, + true + ); + Query expected = new WildcardQuery( + new Term("field" + VALUE_SUFFIX, "field.foo*"), + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + MultiTermQuery.DOC_VALUES_REWRITE + ); + assertEquals(expected, ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + false, + true + ); + Query expected = new WildcardQuery( + new Term("field" + VALUE_AND_PATH_SUFFIX, "field.foo*"), + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + MultiTermQuery.DOC_VALUES_REWRITE + ); + assertEquals(expected, ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 4.test isSearchable=false, hasDocValues=false, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + false, + false + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); + } + + // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + false, + false + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); } } }