From 9078bf333b63a21cfb99e14544b50691ba31aa69 Mon Sep 17 00:00:00 2001 From: iverase Date: Wed, 20 Feb 2019 11:39:05 +0100 Subject: [PATCH 1/5] Extend nextDoc to delegate to the wrapped doc-value iterator for date_nanos. --- .../plain/SortedNumericDVIndexFieldData.java | 5 +++ .../aggregations/bucket/DateHistogramIT.java | 41 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java index 4781a88cecd0c..52adcfe7d1e9f 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java @@ -177,6 +177,11 @@ public long nextValue() throws IOException { public int docValueCount() { return dv.docValueCount(); } + + @Override + public int nextDoc() throws IOException { + return dv.nextDoc(); + } }; } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java index a79797f6c822e..dde227ebcb3b4 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java @@ -1561,4 +1561,45 @@ private void assertMultiSortResponse(int[] expectedDays, BucketOrder... order) { private ZonedDateTime key(Histogram.Bucket bucket) { return (ZonedDateTime) bucket.getKey(); } + + /** + * See https://github.com/elastic/elasticsearch/issues/39107. Make sure we handle properly different + * timeZones. + */ + public void testDateNanosHistogram() throws Exception { + assertAcked(prepareCreate("nanos").addMapping("_doc", "date", "type=date_nanos").get()); + indexRandom(true, + client().prepareIndex("nanos", "_doc", "1").setSource("date", "2000-01-01")); + indexRandom(true, + client().prepareIndex("nanos", "_doc", "2").setSource("date", "2000-01-02")); + + //Search interval 24 hours + SearchResponse r = client().prepareSearch("nanos") + .addAggregation(dateHistogram("histo").field("date").interval(1000 * 60 * 60 * 24).timeZone(ZoneId.of("Europe/Berlin"))) + .addDocValueField("date") + .get(); + assertSearchResponse(r); + + Histogram histogram = r.getAggregations().get("histo"); + List buckets = histogram.getBuckets(); + assertEquals(2, buckets.size()); + assertEquals(946681200000l, ((ZonedDateTime)buckets.get(0).getKey()).toEpochSecond() * 1000); + assertEquals(1, buckets.get(0).getDocCount()); + assertEquals(946767600000l, ((ZonedDateTime)buckets.get(1).getKey()).toEpochSecond() * 1000); + assertEquals(1, buckets.get(1).getDocCount()); + + r = client().prepareSearch("nanos") + .addAggregation(dateHistogram("histo").field("date").interval(1000 * 60 * 60 * 24).timeZone(ZoneId.of("UTC"))) + .addDocValueField("date") + .get(); + assertSearchResponse(r); + + histogram = r.getAggregations().get("histo"); + buckets = histogram.getBuckets(); + assertEquals(2, buckets.size()); + assertEquals(946684800000l, ((ZonedDateTime)buckets.get(0).getKey()).toEpochSecond() * 1000); + assertEquals(1, buckets.get(0).getDocCount()); + assertEquals(946771200000l, ((ZonedDateTime)buckets.get(1).getKey()).toEpochSecond() * 1000); + assertEquals(1, buckets.get(1).getDocCount()); + } } From 92da4b5e9ce885b6576060c0d40ff0848f00c1c6 Mon Sep 17 00:00:00 2001 From: iverase Date: Wed, 20 Feb 2019 11:50:50 +0100 Subject: [PATCH 2/5] fix precommit --- .../search/aggregations/bucket/DateHistogramIT.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java index dde227ebcb3b4..aa59a10e0087a 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java @@ -1575,7 +1575,8 @@ public void testDateNanosHistogram() throws Exception { //Search interval 24 hours SearchResponse r = client().prepareSearch("nanos") - .addAggregation(dateHistogram("histo").field("date").interval(1000 * 60 * 60 * 24).timeZone(ZoneId.of("Europe/Berlin"))) + .addAggregation(dateHistogram("histo").field("date"). + interval(1000 * 60 * 60 * 24).timeZone(ZoneId.of("Europe/Berlin"))) .addDocValueField("date") .get(); assertSearchResponse(r); @@ -1589,7 +1590,8 @@ public void testDateNanosHistogram() throws Exception { assertEquals(1, buckets.get(1).getDocCount()); r = client().prepareSearch("nanos") - .addAggregation(dateHistogram("histo").field("date").interval(1000 * 60 * 60 * 24).timeZone(ZoneId.of("UTC"))) + .addAggregation(dateHistogram("histo").field("date") + .interval(1000 * 60 * 60 * 24).timeZone(ZoneId.of("UTC"))) .addDocValueField("date") .get(); assertSearchResponse(r); From e9dc945530234c0245456b1dbe7a6e8fcd86d213 Mon Sep 17 00:00:00 2001 From: iverase Date: Wed, 20 Feb 2019 11:59:26 +0100 Subject: [PATCH 3/5] fix precommit --- .../search/aggregations/bucket/DateHistogramIT.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java index aa59a10e0087a..26f080f6adb98 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java @@ -1584,9 +1584,9 @@ public void testDateNanosHistogram() throws Exception { Histogram histogram = r.getAggregations().get("histo"); List buckets = histogram.getBuckets(); assertEquals(2, buckets.size()); - assertEquals(946681200000l, ((ZonedDateTime)buckets.get(0).getKey()).toEpochSecond() * 1000); + assertEquals(946681200000L, ((ZonedDateTime)buckets.get(0).getKey()).toEpochSecond() * 1000); assertEquals(1, buckets.get(0).getDocCount()); - assertEquals(946767600000l, ((ZonedDateTime)buckets.get(1).getKey()).toEpochSecond() * 1000); + assertEquals(946767600000L, ((ZonedDateTime)buckets.get(1).getKey()).toEpochSecond() * 1000); assertEquals(1, buckets.get(1).getDocCount()); r = client().prepareSearch("nanos") @@ -1599,9 +1599,9 @@ public void testDateNanosHistogram() throws Exception { histogram = r.getAggregations().get("histo"); buckets = histogram.getBuckets(); assertEquals(2, buckets.size()); - assertEquals(946684800000l, ((ZonedDateTime)buckets.get(0).getKey()).toEpochSecond() * 1000); + assertEquals(946684800000L, ((ZonedDateTime)buckets.get(0).getKey()).toEpochSecond() * 1000); assertEquals(1, buckets.get(0).getDocCount()); - assertEquals(946771200000l, ((ZonedDateTime)buckets.get(1).getKey()).toEpochSecond() * 1000); + assertEquals(946771200000L, ((ZonedDateTime)buckets.get(1).getKey()).toEpochSecond() * 1000); assertEquals(1, buckets.get(1).getDocCount()); } } From bcc7725f523a81b258cc2db5a38c9444ec6b9d56 Mon Sep 17 00:00:00 2001 From: iverase Date: Wed, 20 Feb 2019 17:14:39 +0100 Subject: [PATCH 4/5] Add unit test on DateFieldTypeTests --- .../index/mapper/DateFieldTypeTests.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index 92178e93d212b..500084c44c798 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexOptions; @@ -26,6 +27,8 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.MultiReader; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; @@ -37,6 +40,9 @@ import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.fielddata.AtomicNumericFieldData; +import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.fielddata.plain.SortedNumericDVIndexFieldData; import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType; import org.elasticsearch.index.mapper.MappedFieldType.Relation; import org.elasticsearch.index.mapper.ParseContext.Document; @@ -214,4 +220,36 @@ public void testRangeQuery() throws IOException { () -> ft.rangeQuery(date1, date2, true, true, null, null, null, context)); assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); } + + public void testDateNanoDocValues() throws IOException { + // Create an index with some docValues + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null)); + Document doc = new Document(); + NumericDocValuesField docValuesField = new NumericDocValuesField("my_date", 1444608000000L); + doc.add(docValuesField); + w.addDocument(doc); + docValuesField.setLongValue(1459641600000L); + w.addDocument(doc); + w.flush(); + // Create the doc values reader + Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).build(); + QueryShardContext context = new QueryShardContext(0, + new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(), indexSettings), + null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null); + SortedNumericDVIndexFieldData fieldData = new SortedNumericDVIndexFieldData(context.index(), "my_date", + IndexNumericFieldData.NumericType.DATE_NANOSECONDS); + // Read index and check the doc values + DirectoryReader reader = DirectoryReader.open(w); + assertTrue(reader.leaves().size() > 0); + AtomicNumericFieldData a = fieldData.load(reader.leaves().get(0).reader().getContext()); + SortedNumericDocValues docValues = a.getLongValues(); + assertEquals(0, docValues.nextDoc()); + assertEquals(1, docValues.nextDoc()); + assertEquals(DocIdSetIterator.NO_MORE_DOCS, docValues.nextDoc()); + reader.close(); + w.close(); + dir.close(); + } } From a5e85d0483aaf015795be68d97007a4c23122a3e Mon Sep 17 00:00:00 2001 From: iverase Date: Thu, 21 Feb 2019 09:13:14 +0100 Subject: [PATCH 5/5] clean up unit test --- .../elasticsearch/index/mapper/DateFieldTypeTests.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index 500084c44c798..fab34efbb00e2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -231,14 +231,11 @@ public void testDateNanoDocValues() throws IOException { w.addDocument(doc); docValuesField.setLongValue(1459641600000L); w.addDocument(doc); - w.flush(); // Create the doc values reader - Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).build(); - QueryShardContext context = new QueryShardContext(0, - new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(), indexSettings), - null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null); - SortedNumericDVIndexFieldData fieldData = new SortedNumericDVIndexFieldData(context.index(), "my_date", + IndexSettings indexSettings = new IndexSettings(IndexMetaData.builder("foo").settings(settings).build(), settings); + SortedNumericDVIndexFieldData fieldData = new SortedNumericDVIndexFieldData(indexSettings.getIndex(), "my_date", IndexNumericFieldData.NumericType.DATE_NANOSECONDS); // Read index and check the doc values DirectoryReader reader = DirectoryReader.open(w);