Field caps api - report back if fields are single-valued or not.

Inspects index contents in some cases to reveal if all docs hold single values or not for a field.

Relates to elastic#58523

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java

@@ -56,13 +56,15 @@ public class FieldCapabilities implements Writeable, ToXContentObject {
     private static final ParseField NON_DIMENSION_INDICES_FIELD = new ParseField("non_dimension_indices");
     private static final ParseField METRIC_CONFLICTS_INDICES_FIELD = new ParseField("metric_conflicts_indices");
     private static final ParseField META_FIELD = new ParseField("meta");
+    private static final ParseField IS_SINGLE_VALUED_FIELD = new ParseField("is_single_valued");
 
     private final String name;
     private final String type;
     private final boolean isMetadataField;
     private final boolean isSearchable;
     private final boolean isAggregatable;
     private final boolean isDimension;
+    private final boolean isSingleValued;
     private final TimeSeriesParams.MetricType metricType;
 
     private final String[] indices;
@@ -81,6 +83,7 @@ public class FieldCapabilities implements Writeable, ToXContentObject {
      * @param isSearchable Whether this field is indexed for search.
      * @param isAggregatable Whether this field can be aggregated on.
      * @param isDimension Whether this field can be used as dimension
+     * @param isSingleValued True if all fields hold single values
      * @param metricType If this field is a metric field, returns the metric's type or null for non-metrics fields
      * @param indices The list of indices where this field name is defined as {@code type},
      *                or null if all indices have the same {@code type} for the field.
@@ -99,6 +102,7 @@ public FieldCapabilities(
         boolean isSearchable,
         boolean isAggregatable,
         boolean isDimension,
+        boolean isSingleValued,
         TimeSeriesParams.MetricType metricType,
         String[] indices,
         String[] nonSearchableIndices,
@@ -113,6 +117,7 @@ public FieldCapabilities(
         this.isSearchable = isSearchable;
         this.isAggregatable = isAggregatable;
         this.isDimension = isDimension;
+        this.isSingleValued = isSingleValued;
         this.metricType = metricType;
         this.indices = indices;
         this.nonSearchableIndices = nonSearchableIndices;
@@ -156,6 +161,7 @@ public FieldCapabilities(
             isSearchable,
             isAggregatable,
             false,
+            false,
             null,
             indices,
             nonSearchableIndices,
@@ -175,6 +181,7 @@ public FieldCapabilities(
      * @param isSearchable Whether this field is indexed for search.
      * @param isAggregatable Whether this field can be aggregated on.
      * @param isDimension Whether this field can be used as dimension
+     * @param isSingleValueField True if all documents have only a single value for the field 
      * @param metricType If this field is a metric field, returns the metric's type or null for non-metrics fields
      * @param indices The list of indices where this field name is defined as {@code type},
      *                or null if all indices have the same {@code type} for the field.
@@ -195,6 +202,7 @@ public FieldCapabilities(
         boolean isSearchable,
         boolean isAggregatable,
         Boolean isDimension,
+        Boolean isSingleValueField,
         String metricType,
         List<String> indices,
         List<String> nonSearchableIndices,
@@ -210,6 +218,7 @@ public FieldCapabilities(
             isSearchable,
             isAggregatable,
             isDimension == null ? false : isDimension,
+            isSingleValueField == null ? false : isSingleValueField,
             metricType != null ? Enum.valueOf(TimeSeriesParams.MetricType.class, metricType) : null,
             indices != null ? indices.toArray(new String[0]) : null,
             nonSearchableIndices != null ? nonSearchableIndices.toArray(new String[0]) : null,
@@ -244,6 +253,11 @@ public FieldCapabilities(
             this.metricConflictsIndices = null;
         }
         meta = in.readMap(StreamInput::readString, i -> i.readSet(StreamInput::readString));
+        if (in.getVersion().onOrAfter(Version.V_8_1_0)) {
+            this.isSingleValued = in.readBoolean();
+        } else {
+            this.isSingleValued = false;
+        }        
     }
 
     @Override
@@ -265,6 +279,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalStringArray(metricConflictsIndices);
         }
         out.writeMap(meta, StreamOutput::writeString, (o, set) -> o.writeCollection(set, StreamOutput::writeString));
+        if (out.getVersion().onOrAfter(Version.V_8_1_0)) {
+            out.writeBoolean(isSingleValued);
+        }
     }
 
     @Override
@@ -306,6 +323,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             }
             builder.endObject();
         }
+        builder.field(IS_SINGLE_VALUED_FIELD.getPreferredName(), isSingleValued);
         builder.endObject();
         return builder;
     }
@@ -328,6 +346,7 @@ public static FieldCapabilities fromXContent(String name, XContentParser parser)
         parser.declareBoolean(ConstructingObjectParser.constructorArg(), SEARCHABLE_FIELD);
         parser.declareBoolean(ConstructingObjectParser.constructorArg(), AGGREGATABLE_FIELD);
         parser.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), TIME_SERIES_DIMENSION_FIELD);
+        parser.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), IS_SINGLE_VALUED_FIELD);
         parser.declareString(ConstructingObjectParser.optionalConstructorArg(), TIME_SERIES_METRIC_FIELD);
         parser.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), INDICES_FIELD);
         parser.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_SEARCHABLE_INDICES_FIELD);
@@ -377,6 +396,13 @@ public boolean isDimension() {
         return isDimension;
     }
 
+    /**
+     * True if all documents hold single values.
+     */
+    public boolean isSingleValued() {
+        return isSingleValued;
+    }
+
     /**
      * The metric type
      */
@@ -445,6 +471,7 @@ public boolean equals(Object o) {
             && isSearchable == that.isSearchable
             && isAggregatable == that.isAggregatable
             && isDimension == that.isDimension
+            && isSingleValued == that.isSingleValued
             && Objects.equals(metricType, that.metricType)
             && Objects.equals(name, that.name)
             && Objects.equals(type, that.type)
@@ -458,7 +485,7 @@ public boolean equals(Object o) {
 
     @Override
     public int hashCode() {
-        int result = Objects.hash(name, type, isMetadataField, isSearchable, isAggregatable, isDimension, metricType, meta);
+        int result = Objects.hash(name, type, isMetadataField, isSearchable, isAggregatable, isDimension, isSingleValued, metricType, meta);
         result = 31 * result + Arrays.hashCode(indices);
         result = 31 * result + Arrays.hashCode(nonSearchableIndices);
         result = 31 * result + Arrays.hashCode(nonAggregatableIndices);
@@ -479,6 +506,7 @@ static class Builder {
         private boolean isSearchable;
         private boolean isAggregatable;
         private boolean isDimension;
+        private boolean isSingleValued;
         private TimeSeriesParams.MetricType metricType;
         private boolean metricTypeIsSet;
         private List<IndexCaps> indiceList;
@@ -490,6 +518,7 @@ static class Builder {
             this.isSearchable = true;
             this.isAggregatable = true;
             this.isDimension = true;
+            this.isSingleValued = true;
             this.metricType = null;
             this.metricTypeIsSet = false;
             this.indiceList = new ArrayList<>();
@@ -506,14 +535,16 @@ void add(
             boolean agg,
             boolean isDimension,
             TimeSeriesParams.MetricType metricType,
-            Map<String, String> meta
+            Map<String, String> meta,
+            boolean isSingleValued
         ) {
             IndexCaps indexCaps = new IndexCaps(index, search, agg, isDimension, metricType);
             indiceList.add(indexCaps);
             this.isSearchable &= search;
             this.isAggregatable &= agg;
             this.isMetadataField |= isMetadataField;
             this.isDimension &= isDimension;
+            this.isSingleValued &= isSingleValued;
             // If we have discrepancy in metric types or in some indices this field is not marked as a metric field - we will
             // treat is a non-metric field and report this discrepancy in metricConflictsIndices
             if (this.metricTypeIsSet) {
@@ -598,6 +629,7 @@ FieldCapabilities build(boolean withIndices) {
                 isSearchable,
                 isAggregatable,
                 isDimension,
+                isSingleValued,
                 metricType,
                 indices,
                 nonSearchableIndices,

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java

@@ -86,7 +86,8 @@ FieldCapabilitiesIndexResponse fetch(
                         ft.isAggregatable(),
                         ft.isDimension(),
                         ft.getMetricType(),
-                        ft.meta()
+                        ft.meta(),
+                        ft.isSingleValued(searcher)
                     );
                     responseMap.put(field, fieldCap);
                 } else {
@@ -119,7 +120,10 @@ FieldCapabilitiesIndexResponse fetch(
                                     false,
                                     false,
                                     null,
-                                    Collections.emptyMap()
+                                    Collections.emptyMap(),
+                                    // TODO - perhaps we report on a new object mapper property "allows_multiple_values"
+                                    // that rejects arrays of objects at index time. Here we assume multi-valued always...
+                                    false
                                 );
                                 responseMap.put(parentField, fieldCap);
                             }

server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java

@@ -32,6 +32,7 @@ public class IndexFieldCapabilities implements Writeable {
     private final boolean isSearchable;
     private final boolean isAggregatable;
     private final boolean isDimension;
+    private final boolean isSingleValued;
     private final TimeSeriesParams.MetricType metricType;
     private final Map<String, String> meta;
 
@@ -41,6 +42,7 @@ public class IndexFieldCapabilities implements Writeable {
      * @param isSearchable Whether this field is indexed for search.
      * @param isAggregatable Whether this field can be aggregated on.
      * @param meta Metadata about the field.
+     * @param isSingleValued true if all documents are single-valued
      */
     IndexFieldCapabilities(
         String name,
@@ -50,7 +52,8 @@ public class IndexFieldCapabilities implements Writeable {
         boolean isAggregatable,
         boolean isDimension,
         TimeSeriesParams.MetricType metricType,
-        Map<String, String> meta
+        Map<String, String> meta,
+        boolean isSingleValued
     ) {
 
         this.name = name;
@@ -61,6 +64,7 @@ public class IndexFieldCapabilities implements Writeable {
         this.isDimension = isDimension;
         this.metricType = metricType;
         this.meta = meta;
+        this.isSingleValued = isSingleValued;
     }
 
     IndexFieldCapabilities(StreamInput in) throws IOException {
@@ -77,6 +81,11 @@ public class IndexFieldCapabilities implements Writeable {
             this.metricType = null;
         }
         this.meta = in.readMap(StreamInput::readString, StreamInput::readString);
+        if (in.getVersion().onOrAfter(Version.V_8_1_0)) {
+            this.isSingleValued = in.readBoolean();
+        } else {
+            this.isSingleValued = false;
+        }
     }
 
     @Override
@@ -91,6 +100,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalEnum(metricType);
         }
         out.writeMap(meta, StreamOutput::writeString, StreamOutput::writeString);
+        if (out.getVersion().onOrAfter(Version.V_8_1_0)) {
+            out.writeBoolean(isSingleValued);
+        }
     }
 
     public String getName() {
@@ -117,6 +129,10 @@ public boolean isDimension() {
         return isDimension;
     }
 
+    public boolean isSingleValued() {
+        return isSingleValued;
+    }
+
     public TimeSeriesParams.MetricType getMetricType() {
         return metricType;
     }
@@ -134,6 +150,7 @@ public boolean equals(Object o) {
             && isSearchable == that.isSearchable
             && isAggregatable == that.isAggregatable
             && isDimension == that.isDimension
+            && isSingleValued == that.isSingleValued
             && Objects.equals(metricType, that.metricType)
             && Objects.equals(name, that.name)
             && Objects.equals(type, that.type)
@@ -142,6 +159,6 @@ public boolean equals(Object o) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(name, type, isMetadatafield, isSearchable, isAggregatable, isDimension, metricType, meta);
+        return Objects.hash(name, type, isMetadatafield, isSearchable, isAggregatable, isDimension, isSingleValued, metricType, meta);
     }
 }

server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java

@@ -243,7 +243,7 @@ private void addUnmappedFields(String[] indices, String field, Map<String, Field
             FieldCapabilities.Builder unmapped = new FieldCapabilities.Builder(field, "unmapped");
             typeMap.put("unmapped", unmapped);
             for (String index : unmappedIndices) {
-                unmapped.add(index, false, false, false, false, null, Collections.emptyMap());
+                unmapped.add(index, false, false, false, false, null, Collections.emptyMap(), false);
             }
         }
     }
@@ -271,7 +271,8 @@ private void innerMerge(
                 fieldCap.isAggregatable(),
                 fieldCap.isDimension(),
                 fieldCap.getMetricType(),
-                fieldCap.meta()
+                fieldCap.meta(),
+                fieldCap.isSingleValued()
             );
         }
     }

server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.index.TimestampBounds;
+import org.elasticsearch.index.engine.Engine.Searcher;
 import org.elasticsearch.index.mapper.DateFieldMapper.Resolution;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -45,6 +46,11 @@ public class DataStreamTimestampFieldMapper extends MetadataFieldMapper {
     // In the future it should act as an alias to the actual data stream timestamp field.
     public static final class TimestampFieldType extends MappedFieldType {
 
+        @Override
+        public boolean isSingleValued(Searcher searcher) throws IOException {
+            return true;
+        }
+
         static final TimestampFieldType INSTANCE = new TimestampFieldType();
 
         private TimestampFieldType() {

server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

@@ -31,6 +31,7 @@
 import org.elasticsearch.common.util.LocaleUtils;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.index.engine.Engine.Searcher;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
 import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData;
@@ -692,6 +693,21 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) {
             // TODO maybe aggs should force millis because lots so of other places want nanos?
             return new DocValueFormat.DateTime(dateTimeFormatter, timeZone, Resolution.MILLISECONDS);
         }
+
+        @Override
+        public boolean isSingleValued(Searcher searcher) throws IOException {
+            IndexReader reader = searcher.getTopReaderContext().reader();
+            // Check all segments have one value per doc, exiting early if not.
+            for (LeafReaderContext ctx : reader.leaves()) {
+                PointValues values = ctx.reader().getPointValues(name());
+                if (values != null) {
+                    if (values.size() != values.getDocCount()) {
+                        return false;
+                    }
+                }
+            }
+            return true;
+        }
     }
 
     private final boolean store;

server/src/main/java/org/elasticsearch/index/mapper/DocCountFieldMapper.java

@@ -9,6 +9,7 @@
 
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.index.engine.Engine.Searcher;
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.xcontent.XContentParser;
@@ -73,6 +74,12 @@ protected Object parseSourceValue(Object value) {
                 }
             };
         }
+
+        @Override
+        public boolean isSingleValued(Searcher searcher) throws IOException {
+            return true;
+        }
+
     }
 
     private DocCountFieldMapper() {

server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.fielddata.FieldData;
+import org.elasticsearch.index.engine.Engine.Searcher;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
@@ -208,6 +209,11 @@ public BucketedSort newBucketedSort(
                 }
             };
         }
+
+        @Override
+        public boolean isSingleValued(Searcher searcher) throws IOException {
+            return true;
+        }
     }
 
     private static LeafFieldData wrap(LeafFieldData in) {