diff --git a/docs/changelog/88909.yaml b/docs/changelog/88909.yaml new file mode 100644 index 0000000000000..231871183862d --- /dev/null +++ b/docs/changelog/88909.yaml @@ -0,0 +1,5 @@ +pr: 88909 +summary: Add `synthetic_source` support to `aggregate_metric_double` fields +area: Mapping +type: enhancement +issues: [] diff --git a/docs/reference/mapping/fields/synthetic-source.asciidoc b/docs/reference/mapping/fields/synthetic-source.asciidoc index 875800f8d86c7..32731423a4691 100644 --- a/docs/reference/mapping/fields/synthetic-source.asciidoc +++ b/docs/reference/mapping/fields/synthetic-source.asciidoc @@ -28,6 +28,7 @@ space. There are a couple of restrictions to be aware of: * Synthetic `_source` can be used with indices that contain only these field types: +** <> ** <> ** <> ** <> diff --git a/docs/reference/mapping/types/aggregate-metric-double.asciidoc b/docs/reference/mapping/types/aggregate-metric-double.asciidoc index d6955186a4779..61b4adf2fd029 100644 --- a/docs/reference/mapping/types/aggregate-metric-double.asciidoc +++ b/docs/reference/mapping/types/aggregate-metric-double.asciidoc @@ -251,3 +251,55 @@ The search returns the following hit. The value of the `default_metric` field, } ---- // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,/] + +ifeval::["{release-state}"=="unreleased"] +[[aggregate-metric-double-synthetic-source]] +==== Synthetic source +`aggregate_metric-double` fields support <> in their default +configuration. Synthetic `_source` cannot be used together with <>. + +For example: +[source,console,id=synthetic-source-aggregate-metric-double-example] +---- +PUT idx +{ + "mappings": { + "_source": { "mode": "synthetic" }, + "properties": { + "agg_metric": { + "type": "aggregate_metric_double", + "metrics": [ "min", "max", "sum", "value_count" ], + "default_metric": "max" + } + } + } +} + +PUT idx/_doc/1 +{ + "agg_metric": { + "min": -302.50, + "max": 702.30, + "sum": 200.0, + "value_count": 25 + } +} +---- +// TEST[s/$/\nGET idx\/_doc\/1?filter_path=_source\n/] + +Will become: + +[source,console-result] +---- +{ + "agg_metric": { + "min": -302.50, + "max": 702.30, + "sum": 200.0, + "value_count": 25 + } +} +---- +// TEST[s/^/{"_source":/ s/\n$/}/] + +endif::[] diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 60738a6e2156f..9890e08568784 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -1719,7 +1719,7 @@ protected NumericSyntheticFieldLoader(String name, String simpleName) { @Override public Leaf leaf(LeafReader reader, int[] docIdsInLeaf) throws IOException { - SortedNumericDocValues dv = dv(reader); + SortedNumericDocValues dv = docValuesOrNull(reader, name); if (dv == null) { return SourceLoader.SyntheticFieldLoader.NOTHING_LEAF; } @@ -1830,12 +1830,12 @@ public void write(XContentBuilder b) throws IOException { * an "empty" implementation if there aren't any doc values. We need to be able to * tell if there aren't any and return our empty leaf source loader. */ - private SortedNumericDocValues dv(LeafReader reader) throws IOException { - SortedNumericDocValues dv = reader.getSortedNumericDocValues(name); + public static SortedNumericDocValues docValuesOrNull(LeafReader reader, String fieldName) throws IOException { + SortedNumericDocValues dv = reader.getSortedNumericDocValues(fieldName); if (dv != null) { return dv; } - NumericDocValues single = reader.getNumericDocValues(name); + NumericDocValues single = reader.getNumericDocValues(fieldName); if (single != null) { return DocValues.singleton(single); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 387fa339e65c4..231eaff76d14e 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -253,6 +253,13 @@ protected boolean supportsMeta() { return true; } + /** + * Override to disable testing {@code copy_to} in fields that don't support it. + */ + protected boolean supportsCopyTo() { + return true; + } + protected void metaMapping(XContentBuilder b) throws IOException { minimalMapping(b); } @@ -893,15 +900,17 @@ public final void testSyntheticEmptyList() throws IOException { public final void testSyntheticSourceInvalid() throws IOException { List examples = new ArrayList<>(syntheticSourceSupport().invalidExample()); - examples.add( - new SyntheticSourceInvalidExample( - matchesPattern("field \\[field] of type \\[.+] doesn't support synthetic source because it declares copy_to"), - b -> { - syntheticSourceSupport().example(5).mapping().accept(b); - b.field("copy_to", "bar"); - } - ) - ); + if (supportsCopyTo()) { + examples.add( + new SyntheticSourceInvalidExample( + matchesPattern("field \\[field] of type \\[.+] doesn't support synthetic source because it declares copy_to"), + b -> { + syntheticSourceSupport().example(5).mapping().accept(b); + b.field("copy_to", "bar"); + } + ) + ); + } for (SyntheticSourceInvalidExample example : examples) { Exception e = expectThrows( IllegalArgumentException.class, diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java index f8f9bd9f61483..481f841e88904 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java @@ -9,6 +9,7 @@ import org.apache.lucene.index.DocValues; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.Query; @@ -32,6 +33,7 @@ import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.SimpleMappedFieldType; +import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.TimeSeriesParams; @@ -574,7 +576,6 @@ public Iterator iterator() { @Override protected void parseCreateField(DocumentParserContext context) throws IOException { - context.path().add(simpleName()); XContentParser.Token token; XContentSubParser subParser = null; @@ -675,4 +676,95 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), ignoreMalformedByDefault, indexCreatedVersion).metric(metricType).init(this); } + + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + if (ignoreMalformed) { + throw new IllegalArgumentException( + "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it ignores malformed numbers" + ); + } + return new AggregateMetricSyntheticFieldLoader(name(), simpleName(), metrics); + } + + public static class AggregateMetricSyntheticFieldLoader implements SourceLoader.SyntheticFieldLoader { + private final String name; + private final String simpleName; + private final EnumSet metrics; + + protected AggregateMetricSyntheticFieldLoader(String name, String simpleName, EnumSet metrics) { + this.name = name; + this.simpleName = simpleName; + this.metrics = metrics; + } + + @Override + public Leaf leaf(LeafReader reader, int[] docIdsInLeaf) throws IOException { + Map metricDocValues = new EnumMap<>(Metric.class); + for (Metric m : metrics) { + String fieldName = subfieldName(name, m); + SortedNumericDocValues dv = NumberFieldMapper.NumericSyntheticFieldLoader.docValuesOrNull(reader, fieldName); + if (dv != null) { + metricDocValues.put(m, dv); + } + } + + if (metricDocValues.isEmpty()) { + return SourceLoader.SyntheticFieldLoader.NOTHING_LEAF; + } + + return new AggregateMetricSyntheticFieldLoader.ImmediateLeaf(metricDocValues); + } + + private class ImmediateLeaf implements Leaf { + private final Map metricDocValues; + private final Set metricHasValue = EnumSet.noneOf(Metric.class); + + ImmediateLeaf(Map metricDocValues) { + assert metricDocValues.isEmpty() == false : "doc_values for metrics cannot be empty"; + this.metricDocValues = metricDocValues; + } + + @Override + public boolean empty() { + return false; + } + + @Override + public boolean advanceToDoc(int docId) throws IOException { + // It is required that all defined metrics must exist. In this case + // it is enough to check for the first docValue. However, in the future + // we may relax the requirement of all metrics existing. In this case + // we should check the doc value for each metric separately + metricHasValue.clear(); + for (Map.Entry e : metricDocValues.entrySet()) { + if (e.getValue().advanceExact(docId)) { + metricHasValue.add(e.getKey()); + } + } + + return metricHasValue.isEmpty() == false; + } + + @Override + public void write(XContentBuilder b) throws IOException { + if (metricHasValue.isEmpty()) { + return; + } + b.startObject(simpleName); + for (Map.Entry entry : metricDocValues.entrySet()) { + if (metricHasValue.contains(entry.getKey())) { + String metricName = entry.getKey().name(); + long value = entry.getValue().nextValue(); + if (entry.getKey() == Metric.value_count) { + b.field(metricName, value); + } else { + b.field(metricName, NumericUtils.sortableLongToDouble(value)); + } + } + } + b.endObject(); + } + } + } } diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java index 904effcd6283e..35870fcd7307c 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java @@ -20,12 +20,16 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xpack.aggregatemetric.AggregateMetricMapperPlugin; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Metric; import org.hamcrest.Matchers; import org.junit.AssumptionViolatedException; import java.io.IOException; +import java.util.Arrays; import java.util.Collection; +import java.util.EnumSet; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -33,6 +37,7 @@ import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Names.METRICS; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.matchesPattern; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.core.IsInstanceOf.instanceOf; @@ -393,7 +398,7 @@ public void testExplicitDefaultMetric() throws Exception { Mapper fieldMapper = mapper.mappers().getMapper("field"); assertThat(fieldMapper, instanceOf(AggregateDoubleMetricFieldMapper.class)); - assertEquals(AggregateDoubleMetricFieldMapper.Metric.sum, ((AggregateDoubleMetricFieldMapper) fieldMapper).defaultMetric()); + assertEquals(Metric.sum, ((AggregateDoubleMetricFieldMapper) fieldMapper).defaultMetric()); } /** @@ -406,7 +411,7 @@ public void testImplicitDefaultMetricSingleMetric() throws Exception { Mapper fieldMapper = mapper.mappers().getMapper("field"); assertThat(fieldMapper, instanceOf(AggregateDoubleMetricFieldMapper.class)); - assertEquals(AggregateDoubleMetricFieldMapper.Metric.value_count, ((AggregateDoubleMetricFieldMapper) fieldMapper).defaultMetric); + assertEquals(Metric.value_count, ((AggregateDoubleMetricFieldMapper) fieldMapper).defaultMetric); } /** @@ -416,7 +421,7 @@ public void testImplicitDefaultMetric() throws Exception { DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); Mapper fieldMapper = mapper.mappers().getMapper("field"); assertThat(fieldMapper, instanceOf(AggregateDoubleMetricFieldMapper.class)); - assertEquals(AggregateDoubleMetricFieldMapper.Metric.max, ((AggregateDoubleMetricFieldMapper) fieldMapper).defaultMetric); + assertEquals(Metric.max, ((AggregateDoubleMetricFieldMapper) fieldMapper).defaultMetric); } /** @@ -505,8 +510,8 @@ public void testParseNestedValue() throws Exception { * subfields of aggregate_metric_double should not be searchable or exposed in field_caps */ public void testNoSubFieldsIterated() throws IOException { - AggregateDoubleMetricFieldMapper.Metric[] values = AggregateDoubleMetricFieldMapper.Metric.values(); - List subset = randomSubsetOf(randomIntBetween(1, values.length), values); + Metric[] values = Metric.values(); + List subset = randomSubsetOf(randomIntBetween(1, values.length), values); DocumentMapper mapper = createDocumentMapper( fieldMapping(b -> b.field("type", CONTENT_TYPE).field(METRICS_FIELD, subset).field(DEFAULT_METRIC, subset.get(0))) ); @@ -589,11 +594,58 @@ public void testMetricType() throws IOException { @Override protected SyntheticSourceSupport syntheticSourceSupport() { - throw new AssumptionViolatedException("not supported"); + return new AggregateDoubleMetricSyntheticSourceSupport(); } @Override protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + protected final class AggregateDoubleMetricSyntheticSourceSupport implements SyntheticSourceSupport { + + private final EnumSet storedMetrics = EnumSet.copyOf(randomNonEmptySubsetOf(Arrays.asList(Metric.values()))); + + @Override + public SyntheticSourceExample example(int maxVals) { + // aggregate_metric_double field does not support arrays + Map value = randomAggregateMetric(); + return new SyntheticSourceExample(value, value, this::mapping); + } + + private Map randomAggregateMetric() { + Map value = new LinkedHashMap<>(storedMetrics.size()); + for (Metric m : storedMetrics) { + if (Metric.value_count == m) { + value.put(m.name(), randomLongBetween(1, 1_000_000)); + } else { + value.put(m.name(), randomDouble()); + } + } + return value; + } + + private void mapping(XContentBuilder b) throws IOException { + String[] metrics = storedMetrics.stream().map(Metric::toString).toArray(String[]::new); + b.field("type", CONTENT_TYPE).array(METRICS_FIELD, metrics).field(DEFAULT_METRIC, metrics[0]); + } + + @Override + public List invalidExample() throws IOException { + return List.of( + new SyntheticSourceInvalidExample( + matchesPattern("field \\[field] of type \\[.+] doesn't support synthetic source because it ignores malformed numbers"), + b -> { + mapping(b); + b.field("ignore_malformed", true); + } + ) + ); + } + } + + @Override + protected boolean supportsCopyTo() { + return false; + } } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/100_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/100_synthetic_source.yml new file mode 100644 index 0000000000000..3e6ebdaca9f45 --- /dev/null +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/100_synthetic_source.yml @@ -0,0 +1,53 @@ +constant_keyword: + - skip: + version: " - 8.4.99" + reason: synthetic source support added in 8.5.0 + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + metric: + type: aggregate_metric_double + metrics: [min, max, value_count] + default_metric: max + + - do: + index: + index: test + id: "1" + refresh: false # Do not refresh on every insert so that we get both docs in the same segment + body: + metric: + min: 18.2 + max: 100 + value_count: 50 + + - do: + index: + index: test + id: "2" + refresh: true + body: + metric: + min: 10.0 + max: 20.0 + value_count: 5 + + - do: + search: + index: test + body: + query: + ids: + values: [1, 2] + - match: + hits.hits.0._source: + metric: + min: 18.2 + max: 100.0 + value_count: 50