From 46fcc5805e07fb6a73d70eef90f8bfdcd6ed7d05 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 14 Jul 2020 14:42:08 -0400 Subject: [PATCH] Add tests for keyword script field's fielddata (#59523) Adds tests for the `keyword` typed `script` field's fielddata implementation. One unit test which runs quickly and an integretion test for fetching and an integration test for the `terms` agg. --- .../fielddata/ScriptBinaryFieldData.java | 4 +- .../mapper/RuntimeKeywordMappedFieldType.java | 5 +- .../RuntimeKeywordMappedFieldTypeTests.java | 56 +++++++++++++++++++ .../test/runtime_fields/10_keyword.yml | 27 +++++++++ 4 files changed, 87 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java index 2a21efc52a387..3b17f0b4f9f62 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java @@ -51,7 +51,7 @@ public Builder(StringScriptFieldScript.Factory scriptFactory) { } @Override - public IndexFieldData build( + public ScriptBinaryFieldData build( IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, @@ -130,7 +130,7 @@ public void clear() { } - static class ScriptBinaryLeafFieldData implements LeafFieldData { + public static class ScriptBinaryLeafFieldData implements LeafFieldData { private final ScriptBinaryDocValues scriptBinaryDocValues; ScriptBinaryLeafFieldData(ScriptBinaryDocValues scriptBinaryDocValues) { diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java index c8680da19a8c5..3cf83f3aad33b 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java @@ -8,10 +8,9 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.lucene.BytesRefs; +import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.query.QueryShardContext; @@ -53,7 +52,7 @@ public String typeName() { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public ScriptBinaryFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { // TODO once we get SearchLookup as an argument, we can already call scriptFactory.newFactory here and pass through the result return new ScriptBinaryFieldData.Builder(scriptFactory); } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldTypeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldTypeTests.java index ba09555da47d5..2bd42c2430098 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldTypeTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldTypeTests.java @@ -8,11 +8,21 @@ import org.apache.lucene.document.StoredField; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.Collector; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Scorable; +import org.apache.lucene.search.ScoreMode; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.painless.PainlessPlugin; @@ -25,8 +35,10 @@ import org.elasticsearch.xpack.runtimefields.RuntimeFields; import org.elasticsearch.xpack.runtimefields.RuntimeFieldsPainlessExtension; import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.fielddata.ScriptBinaryFieldData; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import static java.util.Collections.emptyMap; @@ -35,6 +47,50 @@ import static org.mockito.Mockito.when; public class RuntimeKeywordMappedFieldTypeTests extends ESTestCase { + public void testDocValues() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [1]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [2, 1]}")))); + List results = new ArrayList<>(); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + RuntimeKeywordMappedFieldType ft = build("for (def v : source.foo) {value(v.toString())}"); + IndexMetadata imd = IndexMetadata.builder("test") + .settings(Settings.builder().put("index.version.created", Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + ScriptBinaryFieldData ifd = ft.fielddataBuilder("test").build(new IndexSettings(imd, Settings.EMPTY), ft, null, null, null); + ifd.setSearchLookup(mockContext().lookup()); + searcher.search(new MatchAllDocsQuery(), new Collector() { + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE_NO_SCORES; + } + + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + SortedBinaryDocValues dv = ifd.load(context).getBytesValues(); + return new LeafCollector() { + @Override + public void setScorer(Scorable scorer) throws IOException {} + + @Override + public void collect(int doc) throws IOException { + if (dv.advanceExact(doc)) { + for (int i = 0; i < dv.docValueCount(); i++) { + results.add(dv.nextValue().utf8ToString()); + } + } + } + }; + } + }); + assertThat(results, equalTo(List.of("1", "1", "2"))); + } + } + } + public void testTermQuery() throws IOException { try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1}")))); diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml index 5d30364789b5b..241c0d07f6322 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml @@ -40,6 +40,33 @@ setup: {"index":{}} {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"} +--- +"docvalue_fields": + - do: + search: + index: sensor + body: + sort: timestamp + docvalue_fields: [day_of_week] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.day_of_week: [Thursday] } + +--- +"terms agg": + - do: + search: + index: sensor + body: + aggs: + dow: + terms: + field: day_of_week + - match: {hits.total.value: 6} + - match: {aggregations.dow.buckets.0.key: Friday} + - match: {aggregations.dow.buckets.0.doc_count: 1} + - match: {aggregations.dow.buckets.1.key: Monday} + - match: {aggregations.dow.buckets.1.doc_count: 1} + --- "term query": - do: