diff --git a/docs/reference/aggregations/bucket/composite-aggregation.asciidoc b/docs/reference/aggregations/bucket/composite-aggregation.asciidoc index ec253a5556059..4b50536d2ca51 100644 --- a/docs/reference/aggregations/bucket/composite-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/composite-aggregation.asciidoc @@ -287,6 +287,72 @@ Time zones may either be specified as an ISO 8601 UTC offset (e.g. `+01:00` or `-08:00`) or as a timezone id, an identifier used in the TZ database like `America/Los_Angeles`. +*Offset* + +include::datehistogram-aggregation.asciidoc[tag=offset-explanation] + +[source,console,id=composite-aggregation-datehistogram-offset-example] +---- +PUT my_index/_doc/1?refresh +{ + "date": "2015-10-01T05:30:00Z" +} + +PUT my_index/_doc/2?refresh +{ + "date": "2015-10-01T06:30:00Z" +} + +GET my_index/_search?size=0 +{ + "aggs": { + "my_buckets": { + "composite" : { + "sources" : [ + { + "date": { + "date_histogram" : { + "field": "date", + "calendar_interval": "day", + "offset": "+6h", + "format": "iso8601" + } + } + } + ] + } + } + } +} +---- + +include::datehistogram-aggregation.asciidoc[tag=offset-result-intro] + +[source,console-result] +---- +{ + ... + "aggregations": { + "my_buckets": { + "after_key": { "date": "2015-10-01T06:00:00.000Z" }, + "buckets": [ + { + "key": { "date": "2015-09-30T06:00:00.000Z" }, + "doc_count": 1 + }, + { + "key": { "date": "2015-10-01T06:00:00.000Z" }, + "doc_count": 1 + } + ] + } + } +} +---- +// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] + +include::datehistogram-aggregation.asciidoc[tag=offset-note] + ===== Mixing different values source The `sources` parameter accepts an array of values source. diff --git a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc index b4b4a77b32b2d..c925a63beed7d 100644 --- a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc @@ -461,16 +461,19 @@ the bucket covering that day will only hold data for 23 hours instead of the usu where you'll have only a 11h bucket on the morning of 27 March when the DST shift happens. +[[search-aggregations-bucket-datehistogram-offset]] ===== Offset +// tag::offset-explanation[] Use the `offset` parameter to change the start value of each bucket by the specified positive (`+`) or negative offset (`-`) duration, such as `1h` for an hour, or `1d` for a day. See <> for more possible time duration options. For example, when using an interval of `day`, each bucket runs from midnight -to midnight. Setting the `offset` parameter to `+6h` changes each bucket +to midnight. Setting the `offset` parameter to `+6h` changes each bucket to run from 6am to 6am: +// end::offset-explanation[] [source,console,id=datehistogram-aggregation-offset-example] ----------------------------- @@ -498,8 +501,10 @@ GET my_index/_search?size=0 } ----------------------------- +// tag::offset-result-intro[] Instead of a single bucket starting at midnight, the above request groups the documents into buckets starting at 6am: +// end::offset-result-intro[] [source,console-result] ----------------------------- @@ -525,8 +530,10 @@ documents into buckets starting at 6am: ----------------------------- // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] +// tag::offset-note[] NOTE: The start `offset` of each bucket is calculated after `time_zone` adjustments have been made. +// end::offset-note[] ===== Keyed Response diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml index b5e5b8bf41712..12ceb0402bc86 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml @@ -367,6 +367,40 @@ setup: - match: { aggregations.test.buckets.0.key.date: "2017-10-21" } - match: { aggregations.test.buckets.0.doc_count: 1 } +--- +"Composite aggregation with date_histogram offset": + - skip: + version: " - 7.99.99" + reason: offset introduced in 8.0.0 + + - do: + search: + rest_total_hits_as_int: true + index: test + body: + aggregations: + test: + composite: + sources: [ + { + "date": { + "date_histogram": { + "field": "date", + "calendar_interval": "1d", + "offset": "4h", + "format": "iso8601" # Format makes the comparisons a little more obvious + } + } + } + ] + + - match: {hits.total: 6} + - length: { aggregations.test.buckets: 2 } + - match: { aggregations.test.buckets.0.key.date: "2017-10-19T04:00:00.000Z" } + - match: { aggregations.test.buckets.0.doc_count: 1 } + - match: { aggregations.test.buckets.1.key.date: "2017-10-21T04:00:00.000Z" } + - match: { aggregations.test.buckets.1.doc_count: 1 } + --- "Composite aggregation with after_key in the response": - do: diff --git a/server/src/main/java/org/elasticsearch/common/Rounding.java b/server/src/main/java/org/elasticsearch/common/Rounding.java index fa880fafafbe8..daa981e3da843 100644 --- a/server/src/main/java/org/elasticsearch/common/Rounding.java +++ b/server/src/main/java/org/elasticsearch/common/Rounding.java @@ -19,6 +19,7 @@ package org.elasticsearch.common; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -177,6 +178,16 @@ public static Builder builder(TimeValue interval) { return new Builder(interval); } + /** + * Create a rounding that offsets values before passing them into another rounding + * and then un-offsets them after that rounding has done its job. + * @param delegate the other rounding to offset + * @param offset the offset, in milliseconds + */ + public static Rounding offset(Rounding delegate, long offset) { + return new OffsetRounding(delegate, offset); + } + public static class Builder { private final DateTimeUnit unit; @@ -556,19 +567,73 @@ public boolean equals(Object obj) { } } + static class OffsetRounding extends Rounding { + static final byte ID = 3; + + private final Rounding delegate; + private final long offset; + + OffsetRounding(Rounding delegate, long offset) { + this.delegate = delegate; + this.offset = offset; + } + + OffsetRounding(StreamInput in) throws IOException { + delegate = Rounding.read(in); + offset = in.readLong(); + } + + @Override + public void innerWriteTo(StreamOutput out) throws IOException { + if (out.getVersion().before(Version.V_8_0_0)) { + throw new IllegalArgumentException("Offset rounding not supported before 8.0.0"); + } + delegate.writeTo(out); + out.writeLong(offset); + } + + @Override + public byte id() { + return ID; + } + + @Override + public long round(long value) { + return delegate.round(value - offset) + offset; + } + + @Override + public long nextRoundingValue(long value) { + // This isn't needed by the current users. We'll implement it when we migrate other users to it. + throw new UnsupportedOperationException("not yet supported"); + } + + @Override + public int hashCode() { + return Objects.hash(delegate, offset); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || getClass() != obj.getClass()) { + return false; + } + OffsetRounding other = (OffsetRounding) obj; + return delegate.equals(other.delegate) && offset == other.offset; + } + } + public static Rounding read(StreamInput in) throws IOException { - Rounding rounding; byte id = in.readByte(); switch (id) { case TimeUnitRounding.ID: - rounding = new TimeUnitRounding(in); - break; + return new TimeUnitRounding(in); case TimeIntervalRounding.ID: - rounding = new TimeIntervalRounding(in); - break; + return new TimeIntervalRounding(in); + case OffsetRounding.ID: + return new OffsetRounding(in); default: throw new ElasticsearchException("unknown rounding id [" + id + "]"); } - return rounding; } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java index 564399d4c2647..ea35ffe31b577 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.aggregations.bucket.composite; +import org.elasticsearch.Version; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Rounding; import org.elasticsearch.common.io.stream.StreamInput; @@ -31,9 +32,11 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.script.Script; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.search.aggregations.bucket.histogram.DateIntervalConsumer; import org.elasticsearch.search.aggregations.bucket.histogram.DateIntervalWrapper; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; @@ -56,6 +59,13 @@ public class DateHistogramValuesSourceBuilder PARSER = new ObjectParser<>(DateHistogramValuesSourceBuilder.TYPE); PARSER.declareString(DateHistogramValuesSourceBuilder::format, new ParseField("format")); DateIntervalWrapper.declareIntervalFields(PARSER); + PARSER.declareField(DateHistogramValuesSourceBuilder::offset, p -> { + if (p.currentToken() == XContentParser.Token.VALUE_NUMBER) { + return p.longValue(); + } else { + return DateHistogramAggregationBuilder.parseStringOffset(p.text()); + } + }, Histogram.OFFSET_FIELD, ObjectParser.ValueType.LONG); PARSER.declareField(DateHistogramValuesSourceBuilder::timeZone, p -> { if (p.currentToken() == XContentParser.Token.VALUE_STRING) { return ZoneId.of(p.text()); @@ -71,6 +81,7 @@ static DateHistogramValuesSourceBuilder parse(String name, XContentParser parser private ZoneId timeZone = null; private DateIntervalWrapper dateHistogramInterval = new DateIntervalWrapper(); + private long offset = 0; public DateHistogramValuesSourceBuilder(String name) { super(name, ValueType.DATE); @@ -80,12 +91,18 @@ protected DateHistogramValuesSourceBuilder(StreamInput in) throws IOException { super(in); dateHistogramInterval = new DateIntervalWrapper(in); timeZone = in.readOptionalZoneId(); + if (in.getVersion().after(Version.V_8_0_0)) { + offset = in.readLong(); + } } @Override protected void innerWriteTo(StreamOutput out) throws IOException { dateHistogramInterval.writeTo(out); out.writeOptionalZoneId(timeZone); + if (out.getVersion().after(Version.V_8_0_0)) { + out.writeLong(offset); + } } @Override @@ -215,9 +232,28 @@ public ZoneId timeZone() { return timeZone; } + /** + * Get the offset to use when rounding, which is a number of milliseconds. + */ + public long offset() { + return offset; + } + + /** + * Set the offset on this builder, which is a number of milliseconds. + * @return this for chaining + */ + public DateHistogramValuesSourceBuilder offset(long offset) { + this.offset = offset; + return this; + } + @Override protected CompositeValuesSourceConfig innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config) throws IOException { Rounding rounding = dateHistogramInterval.createRounding(timeZone()); + if (offset != 0) { + rounding = Rounding.offset(rounding, offset); + } ValuesSource orig = config.toValuesSource(queryShardContext); if (orig == null) { orig = ValuesSource.Numeric.EMPTY; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java index 96e5641619caa..81c697230b4bd 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java @@ -285,7 +285,10 @@ public DateHistogramAggregationBuilder offset(String offset) { return offset(parseStringOffset(offset)); } - static long parseStringOffset(String offset) { + /** + * Parse the string specification of an offset. + */ + public static long parseStringOffset(String offset) { if (offset.charAt(0) == '-') { return -TimeValue .parseTimeValue(offset.substring(1), null, DateHistogramAggregationBuilder.class.getSimpleName() + ".parseOffset") diff --git a/server/src/test/java/org/elasticsearch/common/RoundingTests.java b/server/src/test/java/org/elasticsearch/common/RoundingTests.java index bee3f57764f32..26401a392f3e3 100644 --- a/server/src/test/java/org/elasticsearch/common/RoundingTests.java +++ b/server/src/test/java/org/elasticsearch/common/RoundingTests.java @@ -195,6 +195,16 @@ public void testTimeUnitRoundingDST() { assertThat(tzRounding_chg.round(time("2014-11-02T06:01:01", chg)), isDate(time("2014-11-02T06:00:00", chg), chg)); } + public void testOffsetRounding() { + long twoHours = TimeUnit.HOURS.toMillis(2); + long oneDay = TimeUnit.DAYS.toMillis(1); + Rounding dayOfMonth = Rounding.builder(Rounding.DateTimeUnit.DAY_OF_MONTH).build(); + Rounding offsetRounding = Rounding.offset(dayOfMonth, twoHours); + assertThat(offsetRounding.round(0), equalTo(-oneDay + twoHours)); + + assertThat(offsetRounding.round(twoHours), equalTo(twoHours)); + } + /** * Randomized test on TimeUnitRounding. Test uses random * {@link DateTimeUnit} and {@link ZoneId} and often (50% of the time) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java index 601154234e792..7769dee3c5833 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java @@ -90,6 +90,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; @@ -1020,7 +1021,7 @@ public void testWithDateHistogram() throws IOException { () -> { DateHistogramValuesSourceBuilder histo = new DateHistogramValuesSourceBuilder("date") .field("date") - .dateHistogramInterval(DateHistogramInterval.days(1)); + .calendarInterval(DateHistogramInterval.days(1)); return new CompositeAggregationBuilder("name", Collections.singletonList(histo)); }, (result) -> { @@ -1044,7 +1045,7 @@ public void testWithDateHistogram() throws IOException { () -> { DateHistogramValuesSourceBuilder histo = new DateHistogramValuesSourceBuilder("date") .field("date") - .dateHistogramInterval(DateHistogramInterval.days(1)); + .calendarInterval(DateHistogramInterval.days(1)); return new CompositeAggregationBuilder("name", Collections.singletonList(histo)) .aggregateAfter(createAfterKey("date", 1474329600000L)); @@ -1058,7 +1059,35 @@ public void testWithDateHistogram() throws IOException { } ); - assertWarnings("[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future."); + /* + * Tests a four hour offset, which moves the document with + * date 2017-10-20T03:08:45 into 2017-10-19's bucket. + */ + testSearchCase(Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery("date"), + LongPoint.newRangeQuery( + "date", + asLong("2016-09-20T09:00:34"), + asLong("2017-10-20T06:09:24") + )), dataset, + () -> { + DateHistogramValuesSourceBuilder histo = new DateHistogramValuesSourceBuilder("date") + .field("date") + .calendarInterval(DateHistogramInterval.days(1)) + .offset(TimeUnit.HOURS.toMillis(4)); + return new CompositeAggregationBuilder("name", Collections.singletonList(histo)) + .aggregateAfter(createAfterKey("date", 1474329600000L)); + + }, (result) -> { + assertEquals(3, result.getBuckets().size()); + assertEquals("{date=1508472000000}", result.afterKey().toString()); + assertEquals("{date=1474344000000}", result.getBuckets().get(0).getKeyAsString()); + assertEquals(2L, result.getBuckets().get(0).getDocCount()); + assertEquals("{date=1508385600000}", result.getBuckets().get(1).getKeyAsString()); + assertEquals(2L, result.getBuckets().get(1).getDocCount()); + assertEquals("{date=1508472000000}", result.getBuckets().get(2).getKeyAsString()); + assertEquals(1L, result.getBuckets().get(2).getDocCount()); + } + ); } public void testWithDateTerms() throws IOException {