Add support for flattened datatype (#4058)

* Add support for flattened datatype

Relates #4001

This commit adds support for the flattened data type introduced in 7.3.0 and basic.

(cherry picked from commit fe9b212)

src/Nest/Mapping/Types/Complex/Flattened/FlattenedAttribute.cs

@@ -0,0 +1,83 @@
+using System.Xml.Schema;
+
+namespace Nest
+{
+	/// <inheritdoc cref="IFlattenedProperty" />
+	public class FlattenedAttribute : ElasticsearchPropertyAttributeBase, IFlattenedProperty
+	{
+		public FlattenedAttribute() : base(FieldType.Flattened) { }
+
+		private IFlattenedProperty Self => this;
+
+		double? IFlattenedProperty.Boost { get; set; }
+		int? IFlattenedProperty.DepthLimit { get; set; }
+		bool? IFlattenedProperty.DocValues { get; set; }
+		bool? IFlattenedProperty.EagerGlobalOrdinals { get; set; }
+		int? IFlattenedProperty.IgnoreAbove { get; set; }
+		bool? IFlattenedProperty.Index { get; set; }
+		IndexOptions? IFlattenedProperty.IndexOptions { get; set; }
+		bool? IFlattenedProperty.SplitQueriesOnWhitespace { get; set; }
+
+		/// <inheritdoc cref="IFlattenedProperty.Boost" />
+		public double Boost
+		{
+			get => Self.Boost.GetValueOrDefault(1);
+			set => Self.Boost = value;
+		}
+
+		/// <inheritdoc cref="IFlattenedProperty.DepthLimit" />
+		public int DepthLimit
+		{
+			get => Self.DepthLimit.GetValueOrDefault(20);
+			set => Self.DepthLimit = value;
+		}
+
+		/// <inheritdoc cref="IFlattenedProperty.DocValues" />
+		public bool DocValues
+		{
+			get => Self.DocValues.GetValueOrDefault(true);
+			set => Self.DocValues = value;
+		}
+
+		/// <inheritdoc cref="IFlattenedProperty.EagerGlobalOrdinals" />
+		public bool EagerGlobalOrdinals
+		{
+			get => Self.EagerGlobalOrdinals.GetValueOrDefault(false);
+			set => Self.EagerGlobalOrdinals = value;
+		}
+
+		/// <inheritdoc cref="IFlattenedProperty.IgnoreAbove" />
+		public int IgnoreAbove
+		{
+			get => Self.IgnoreAbove.GetValueOrDefault(-1);
+			set => Self.IgnoreAbove = value;
+		}
+
+		/// <inheritdoc cref="IFlattenedProperty.Index" />
+		public bool Index
+		{
+			get => Self.Index.GetValueOrDefault(true);
+			set => Self.Index = value;
+		}
+
+		/// <inheritdoc cref="IFlattenedProperty.IndexOptions" />
+		public IndexOptions IndexOptions
+		{
+			get => Self.IndexOptions.GetValueOrDefault(IndexOptions.Docs);
+			set => Self.IndexOptions = value;
+		}
+
+		/// <inheritdoc cref="IFlattenedProperty.SplitQueriesOnWhitespace" />
+		public bool SplitQueriesOnWhitespace
+		{
+			get => Self.SplitQueriesOnWhitespace.GetValueOrDefault(false);
+			set => Self.SplitQueriesOnWhitespace = value;
+		}
+
+		/// <inheritdoc />
+		public string NullValue { get; set; }
+
+		/// <inheritdoc />
+		public string Similarity { get; set; }
+	}
+}

src/Nest/Mapping/Types/Complex/Flattened/FlattenedProperty.cs

@@ -0,0 +1,175 @@
+using System.Diagnostics;
+using System.Runtime.Serialization;
+using Elasticsearch.Net.Utf8Json;
+
+namespace Nest
+{
+	/// <summary>
+	/// By default, each subfield in an object is mapped and indexed separately.
+	/// If the names or types of the subfields are not known in advance, then they are mapped dynamically.
+	/// The flattened type provides an alternative approach, where the entire object is mapped as a single field.
+	/// Given an object, the flattened mapping will parse out its leaf values and index them into one
+	/// field as keywords. The object's contents can then be searched through simple queries and aggregations.
+	/// <para></para>
+	/// Available in Elasticsearch 7.3.0+ with at least basic license level
+	/// </summary>
+	[InterfaceDataContract]
+	public interface IFlattenedProperty : IProperty
+	{
+		/// <summary>
+		/// Mapping field-level query time boosting. Accepts a floating point number, defaults to 1.0.
+		/// </summary>
+		[DataMember(Name = "boost")]
+		double? Boost { get; set; }
+
+		/// <summary>
+		/// The maximum allowed depth of the flattened object field,
+		/// in terms of nested inner objects. If a flattened object field exceeds this limit,
+		/// then an error will be thrown. Defaults to <c>20</c>.
+		/// </summary>
+		[DataMember(Name = "depth_limit")]
+		int? DepthLimit { get; set; }
+
+		/// <summary>
+		/// Whether to persist the value at index time in a columnar data structure (referred to as doc_values in Lucene)
+		/// which makes the value available for efficient sorting and aggregations. Default is <c>true</c>.
+		/// </summary>
+		[DataMember(Name = "doc_values")]
+		bool? DocValues { get; set; }
+
+		/// <summary>
+		/// Should global ordinals be loaded eagerly on refresh? Accepts true or false (default).
+		/// Enabling this is a good idea on fields that are frequently used for terms aggregations.
+		/// </summary>
+		[DataMember(Name = "eager_global_ordinals")]
+		bool? EagerGlobalOrdinals { get; set; }
+
+		/// <summary>
+		/// Leaf values longer than this limit will not be indexed. By default, there is no limit and all values will be indexed.
+		/// Note that this limit applies to the leaf values within the flattened object field, and not the length of the entire
+		/// field.
+		/// </summary>
+		[DataMember(Name = "ignore_above")]
+		int? IgnoreAbove { get; set; }
+
+		/// <summary>
+		/// Should the field be searchable? Accepts <c>true</c> (default) and <c>false</c>.
+		/// </summary>
+		[DataMember(Name = "index")]
+		bool? Index { get; set; }
+
+		/// <summary>
+		/// What information should be stored in the index for scoring purposes.
+		/// Defaults to docs but can also be set to freqs to take term frequency into account when computing scores.
+		/// </summary>
+		[DataMember(Name = "index_options")]
+		IndexOptions? IndexOptions { get; set; }
+
+		/// <summary>
+		/// A string value which is substituted for any explicit null values within the flattened
+		/// object field. Defaults to null, which means null fields are treated as if it were missing.
+		/// </summary>
+		[DataMember(Name = "null_value")]
+		string NullValue { get; set; }
+
+		/// <summary>
+		/// Which relevancy scoring algorithm or similarity should be used.
+		/// Defaults to BM25
+		/// </summary>
+		[DataMember(Name = "similarity")]
+		string Similarity { get; set; }
+
+		/// <summary> Whether full text queries should split the input on whitespace when building a query for this field. </summary>
+		[DataMember(Name = "split_queries_on_whitespace")]
+		bool? SplitQueriesOnWhitespace { get; set; }
+	}
+
+	/// <inheritdoc cref="IFlattenedProperty" />
+	[DebuggerDisplay("{DebugDisplay}")]
+	public class FlattenedProperty : PropertyBase, IFlattenedProperty
+	{
+		public FlattenedProperty() : base(FieldType.Flattened) { }
+
+		/// <inheritdoc />
+		public double? Boost { get; set; }
+
+		/// <inheritdoc />
+		public int? DepthLimit { get; set; }
+
+		/// <inheritdoc />
+		public bool? DocValues { get; set; }
+
+		/// <inheritdoc />
+		public bool? EagerGlobalOrdinals { get; set; }
+
+		/// <inheritdoc />
+		public int? IgnoreAbove { get; set; }
+
+		/// <inheritdoc />
+		public bool? Index { get; set; }
+
+		/// <inheritdoc />
+		public IndexOptions? IndexOptions { get; set; }
+
+		/// <inheritdoc />
+		public string NullValue { get; set; }
+
+		/// <inheritdoc />
+		public string Similarity { get; set; }
+
+		/// <inheritdoc />
+		public bool? SplitQueriesOnWhitespace { get; set; }
+	}
+
+	/// <inheritdoc cref="IFlattenedProperty" />
+	[DebuggerDisplay("{DebugDisplay}")]
+	public class FlattenedPropertyDescriptor<T>
+		: PropertyDescriptorBase<FlattenedPropertyDescriptor<T>, IFlattenedProperty, T>, IFlattenedProperty
+		where T : class
+	{
+		public FlattenedPropertyDescriptor() : base(FieldType.Flattened) { }
+
+		double? IFlattenedProperty.Boost { get; set; }
+		int? IFlattenedProperty.DepthLimit { get; set; }
+		bool? IFlattenedProperty.DocValues { get; set; }
+		bool? IFlattenedProperty.EagerGlobalOrdinals { get; set; }
+		int? IFlattenedProperty.IgnoreAbove { get; set; }
+		bool? IFlattenedProperty.Index { get; set; }
+		IndexOptions? IFlattenedProperty.IndexOptions { get; set; }
+		string IFlattenedProperty.NullValue { get; set; }
+		string IFlattenedProperty.Similarity { get; set; }
+		bool? IFlattenedProperty.SplitQueriesOnWhitespace { get; set; }
+
+		/// <inheritdoc cref="IFlattenedProperty.Boost" />
+		public FlattenedPropertyDescriptor<T> Boost(double? boost) => Assign(boost, (a, v) => a.Boost = v);
+
+		/// <inheritdoc cref="IFlattenedProperty.DepthLimit" />
+		public FlattenedPropertyDescriptor<T> DepthLimit(int? depthLimit) => Assign(depthLimit, (a, v) => a.DepthLimit = v);
+
+		/// <inheritdoc cref="IFlattenedProperty.DepthLimit" />
+		public FlattenedPropertyDescriptor<T> DocValues(bool? docValues = true) => Assign(docValues, (a, v) => a.DocValues = v);
+
+		/// <inheritdoc cref="IFlattenedProperty.EagerGlobalOrdinals" />
+		public FlattenedPropertyDescriptor<T> EagerGlobalOrdinals(bool? eagerGlobalOrdinals = true) =>
+			Assign(eagerGlobalOrdinals, (a, v) => a.EagerGlobalOrdinals = v);
+
+		/// <inheritdoc cref="IFlattenedProperty.IgnoreAbove" />
+		public FlattenedPropertyDescriptor<T> IgnoreAbove(int? ignoreAbove) => Assign(ignoreAbove, (a, v) => a.IgnoreAbove = v);
+
+		/// <inheritdoc cref="IFlattenedProperty.Index" />
+		public FlattenedPropertyDescriptor<T> Index(bool? index = true) => Assign(index, (a, v) => a.Index = v);
+
+		/// <inheritdoc cref="IFlattenedProperty.IndexOptions" />
+		public FlattenedPropertyDescriptor<T> IndexOptions(IndexOptions? indexOptions) => Assign(indexOptions, (a, v) => a.IndexOptions = v);
+
+		/// <inheritdoc cref="IFlattenedProperty.NullValue" />
+		public FlattenedPropertyDescriptor<T> NullValue(string nullValue) => Assign(nullValue, (a, v) => a.NullValue = v);
+
+		/// <inheritdoc cref="IFlattenedProperty.SplitQueriesOnWhitespace" />
+		public FlattenedPropertyDescriptor<T> SplitQueriesOnWhitespace(bool? splitQueriesOnWhitespace = true) =>
+			Assign(splitQueriesOnWhitespace, (a, v) => a.SplitQueriesOnWhitespace = v);
+
+		/// <inheritdoc cref="IFlattenedProperty.Similarity" />
+		public FlattenedPropertyDescriptor<T> Similarity(string similarity) => Assign(similarity, (a, v) => a.Similarity = v);
+	}
+}

src/Nest/Mapping/Types/FieldType.cs

@@ -132,6 +132,9 @@ public enum FieldType
 		RankFeature,
 
 		[EnumMember(Value = "rank_features")]
-		RankFeatures
+		RankFeatures,
+
+		[EnumMember(Value = "flattened")]
+		Flattened
 	}
 }

src/Nest/Mapping/Types/Properties.cs

@@ -166,6 +166,9 @@ public PropertiesDescriptor<T> Object<TChild>(Func<ObjectTypeDescriptor<T, TChil
 		/// <inheritdoc cref="IRankFeaturesProperty"/>
 		public PropertiesDescriptor<T> RankFeatures(Func<RankFeaturesPropertyDescriptor<T>, IRankFeaturesProperty> selector) => SetProperty(selector);
 
+		/// <inheritdoc cref="IFlattenedProperty"/>
+		public PropertiesDescriptor<T> Flattened(Func<FlattenedPropertyDescriptor<T>, IFlattenedProperty> selector) => SetProperty(selector);
+
 		public PropertiesDescriptor<T> Custom(IProperty customType) => SetProperty(customType);
 
 		private PropertiesDescriptor<T> SetProperty<TDescriptor, TInterface>(Func<TDescriptor, TInterface> selector)

src/Nest/Mapping/Types/PropertyFormatter.cs

@@ -91,6 +91,7 @@ public IProperty Deserialize(ref JsonReader reader, IJsonFormatterResolver forma
 				case FieldType.Alias: return Deserialize<FieldAliasProperty>(ref segmentReader, formatterResolver);
 				case FieldType.RankFeature: return Deserialize<RankFeatureProperty>(ref segmentReader, formatterResolver);
 				case FieldType.RankFeatures: return Deserialize<RankFeaturesProperty>(ref segmentReader, formatterResolver);
+				case FieldType.Flattened: return Deserialize<FlattenedProperty>(ref segmentReader, formatterResolver);
 				case FieldType.None:
 					// no "type" field in the property mapping
 					return Deserialize<ObjectProperty>(ref segmentReader, formatterResolver);
@@ -197,6 +198,9 @@ public void Serialize(ref JsonWriter writer, IProperty value, IJsonFormatterReso
 				case "rank_features":
 					Serialize<IRankFeaturesProperty>(ref writer, value, formatterResolver);
 					break;
+				case "flattened":
+					Serialize<IFlattenedProperty>(ref writer, value, formatterResolver);
+					break;
 				default:
 					if (value is IGenericProperty genericProperty)
 						Serialize<IGenericProperty>(ref writer, genericProperty, formatterResolver);

src/Nest/Mapping/Visitor/IMappingVisitor.cs

@@ -57,6 +57,8 @@ public interface IMappingVisitor
 		void Visit(IRankFeaturesProperty property);
 
 		void Visit(ISearchAsYouTypeProperty property);
+
+		void Visit(IFlattenedProperty property);
 	}
 
 	public class NoopMappingVisitor : IMappingVisitor
@@ -116,5 +118,7 @@ public virtual void Visit(IRankFeatureProperty property) { }
 		public virtual void Visit(IRankFeaturesProperty property) { }
 
 		public virtual void Visit(ISearchAsYouTypeProperty property) { }
+
+		public virtual void Visit(IFlattenedProperty property) { }
 	}
 }

src/Nest/Mapping/Visitor/IPropertyVisitor.cs

@@ -56,6 +56,8 @@ public interface IPropertyVisitor
 
 		void Visit(IProperty type, PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute);
 
+		void Visit(IFlattenedProperty type, PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute);
+
 		IProperty Visit(PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute);
 
 		bool SkipProperty(PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute);

src/Nest/Mapping/Visitor/MappingWalker.cs

@@ -223,7 +223,10 @@ public void Accept(IProperties properties)
 						});
 						break;
 					case FieldType.Join:
-						Visit<IJoinProperty>(field, t => { _visitor.Visit(t); });
+						Visit<IJoinProperty>(field, t =>
+						{
+							_visitor.Visit(t);
+						});
 						break;
 					case FieldType.RankFeature:
 						Visit<IRankFeatureProperty>(field, t =>
@@ -237,6 +240,12 @@ public void Accept(IProperties properties)
 							_visitor.Visit(t);
 						});
 						break;
+					case FieldType.Flattened:
+						Visit<IFlattenedProperty>(field, t =>
+						{
+							_visitor.Visit(t);
+						});
+						break;
 				}
 			}
 		}

src/Nest/Mapping/Visitor/NoopPropertyVisitor.cs

@@ -56,6 +56,8 @@ public virtual void Visit(ITextProperty type, PropertyInfo propertyInfo, Elastic
 
 		public virtual void Visit(IKeywordProperty type, PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute) { }
 
+		public virtual void Visit(IFlattenedProperty type, PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute) { }
+
 		public virtual IProperty Visit(PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute) => null;
 
 		public void Visit(IProperty type, PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute)