Set ECS fields from message templates

src/Elastic.CommonSchema.Generator/Elastic.CommonSchema.Generator.csproj

@@ -9,12 +9,12 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="RazorLight" Version="2.3.0" />
     <PackageReference Include="Cogito.Json.Schema.Validation" Version="1.0.1" />
     <PackageReference Include="CsQuery.Core" Version="2.0.1" />
     <PackageReference Include="JsonDiffPatch" Version="2.0.49" />
     <PackageReference Include="JsonDiffPatch.Net" Version="2.1.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
-    <PackageReference Include="RazorLight.Unofficial" Version="2.0.0-beta1.3" />
     <PackageReference Include="ShellProgressBar" Version="5.0.0" />
     <PackageReference Include="YamlDotNet" Version="6.0.0" />
   </ItemGroup>

src/Elastic.CommonSchema.Generator/FileGenerator.cs

@@ -26,6 +26,8 @@ public static void Generate(CommonSchemaTypesProjection commonSchemaTypesProject
 			{
 				{ m => Generate(m, "EcsDocument"), "Base ECS Document" },
 				{ m => Generate(m, "EcsDocumentJsonConverter"), "Base ECS Document Json Converter" },
+				{ m => Generate(m, "LogTemplateProperties"), "Strongly types ECS fields supported in log templates" },
+				{ m => Generate(m, "PropDispatch"), "ECS key value setter generation" },
 				{ m => Generate(m, "EcsJsonContext"), "Ecs System Text Json Source Generators" },
 				{ m => Generate(m, "FieldSets"), "Field Sets" },
 				{ m => Generate(m, "Entities"), "Entities" },
@@ -35,7 +37,7 @@ public static void Generate(CommonSchemaTypesProjection commonSchemaTypesProject
 			};
 
 			using (var progressBar = new ProgressBar(actions.Count, "Generating code",
-				new ProgressBarOptions { BackgroundColor = ConsoleColor.DarkGray }))
+					   new ProgressBarOptions { BackgroundColor = ConsoleColor.DarkGray }))
 			{
 				foreach (var kv in actions)
 				{

src/Elastic.CommonSchema.Generator/Projection/ProjectionTypeExtensions.cs

@@ -11,6 +11,7 @@ public static class ProjectionTypeExtensions
 		public static string PascalCase(this string s) => new CultureInfo("en-US")
 			.TextInfo
 			.ToTitleCase(s.ToLowerInvariant())
+			.Replace("@", string.Empty)
 			.Replace("_", string.Empty)
 			.Replace(".", string.Empty);
 
@@ -23,6 +24,35 @@ public static string GetClrType(this Field field)
 			return field.Normalize.Contains("array") ? $"{baseType}[]" : baseType;
 		}
 
+		public static string GetCastFromObject(this Field field)
+		{
+			if (field.Normalize.Contains("array")) return null;
+			switch (field.Type)
+			{
+				case FieldType.Keyword:
+				case FieldType.ConstantKeyword:
+				case FieldType.Flattened:
+				case FieldType.MatchOnlyText:
+				case FieldType.Wildcard:
+				case FieldType.Text:
+				case FieldType.Ip:
+					return "TrySetString";
+				case FieldType.Boolean:
+					return "TrySetBool";
+				case FieldType.ScaledFloat:
+				case FieldType.Float:
+					return "TrySetFloat";
+				case FieldType.Long:
+					return "TrySetLong";
+				case FieldType.Integer:
+					return "TrySetInt";
+				case FieldType.Date:
+					return "TrySetDateTimeOffset";
+				default: return null;
+			}
+
+		}
+
 		private static string GetClrType(this FieldType fieldType)
 		{
 			switch (fieldType)

src/Elastic.CommonSchema.Generator/Projection/PropertyReference.cs

@@ -18,6 +18,7 @@ protected PropertyReference(string localPath, string fullPath)
 
 		public string LocalPath { get; }
 		public string FullPath { get; }
+		public string LogTemplateAlternative => FullPath.PascalCase();
 
 		public abstract string Description { get; }
 		public abstract string Example { get; }
@@ -79,11 +80,12 @@ public class ValueTypePropertyReference : PropertyReference
 		public ValueTypePropertyReference(string parentPath, string fullPath, Field field) : base(parentPath, fullPath)
 		{
 			ClrType = field.GetClrType();
+			CastFromObject = field.GetCastFromObject();
 			Description = GetFieldDescription(field);
 			Example = field.Example?.ToString() ?? string.Empty;
-
 		}
 
+		public string CastFromObject { get; }
 		public string ClrType { get; }
 		public override string Description { get; }
 		public override string Example { get; }

src/Elastic.CommonSchema.Generator/Projection/Types.cs

@@ -13,6 +13,9 @@ public class FieldSetBaseClass
 
 		public Dictionary<string, PropertyReference> Properties { get; } = new();
 
+		public IEnumerable<ValueTypePropertyReference> SettableProperties =>
+			ValueProperties.Where(p => !string.IsNullOrEmpty(p.CastFromObject));
+
 		public IEnumerable<ValueTypePropertyReference> ValueProperties =>
 			Properties.Values.OfType<ValueTypePropertyReference>();
 

src/Elastic.CommonSchema.Generator/Views/EcsDocument.Generated.cshtml

@@ -36,6 +36,10 @@ namespace Elastic.CommonSchema
 
 		/// <summary>
 		/// Container for additional metadata against this event.
+		/// <para/>
+		/// When working with unknown fields use <see cref="SetAnyField"/>. <br/>
+		/// <para> This will try to assign valid ECS fields to their respective property 
+		/// Failing that it will assign strings to <see cref="Labels"/> and everything else to <see cref="Metadata"/> </para>
 		/// </summary>
 		[JsonPropertyName("metadata"), DataMember(Name = "metadata")]
 		[JsonConverter(typeof(MetadataDictionaryConverter))]

src/Elastic.CommonSchema.Generator/Views/LogTemplateProperties.Generated.cshtml

@@ -0,0 +1,67 @@
+@using RazorLight
+@using System
+@using Generator
+@inherits Elastic.CommonSchema.Generator.Views.CodeTemplatePage<Elastic.CommonSchema.Generator.Projection.CommonSchemaTypesProjection>
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+/*
+IMPORTANT NOTE
+==============
+This file has been generated. 
+If you wish to submit a PR please modify the original csharp file and submit the PR with that change. Thanks!
+*/
+
+// ReSharper disable RedundantUsingDirective
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Linq;
+using System.Net;
+using System.Runtime.Serialization;
+using System.Text.Json.Serialization;
+using Elastic.CommonSchema.Serialization;
+using static Elastic.CommonSchema.PropDispatch;
+
+namespace Elastic.CommonSchema
+{
+	public static class LogTemplateProperties
+	{
+@foreach (var prop in Model.Base.BaseFieldSet.SettableProperties)
+{
+<text>		public static string @prop.LogTemplateAlternative = nameof(@prop.LogTemplateAlternative);
+</text>
+}
+@foreach (var entity in Model.EntityClasses)
+{
+	@foreach (var prop in entity.BaseFieldSet.SettableProperties)
+	{
+<text>		public static string @prop.LogTemplateAlternative = nameof(@prop.LogTemplateAlternative);
+</text>
+
+	}
+}
+
+		public static readonly HashSet@(Raw("<string>")) All = new()
+		{
+@foreach (var prop in Model.Base.BaseFieldSet.SettableProperties)
+{
+<text>			"@prop.FullPath", @prop.LogTemplateAlternative,
+</text>
+
+}
+@foreach (var entity in Model.EntityClasses)
+{
+	@foreach (var prop in entity.BaseFieldSet.SettableProperties)
+	{
+<text>			"@prop.FullPath", @prop.LogTemplateAlternative,
+</text>
+
+	}
+}
+		};
+	}
+
+}

src/Elastic.CommonSchema.Generator/Views/PropDispatch.Generated.cshtml

@@ -0,0 +1,128 @@
+@using RazorLight
+@using System
+@using Generator
+@using System.Linq;
+@inherits Elastic.CommonSchema.Generator.Views.CodeTemplatePage<Elastic.CommonSchema.Generator.Projection.CommonSchemaTypesProjection>
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+/*
+IMPORTANT NOTE
+==============
+This file has been generated. 
+If you wish to submit a PR please modify the original csharp file and submit the PR with that change. Thanks!
+*/
+
+// ReSharper disable RedundantUsingDirective
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Linq;
+using System.Net;
+using System.Runtime.Serialization;
+using System.Text.Json.Serialization;
+using Elastic.CommonSchema.Serialization;
+using static Elastic.CommonSchema.PropDispatch;
+
+namespace Elastic.CommonSchema
+{
+	///<inheritdoc cref="@Model.Base.BaseFieldSet.Name"/>
+	public partial class @Model.Base.Name : @Model.Base.BaseFieldSet.Name 
+	{
+		/// <summary>
+		/// Set ECS fields by name on <see cref="EcsDocument"/>.
+		/// <para>Allows valid ECS fields to be set from log message templates.</para>
+		/// Given <paramref name="value"/>'s type matches the corresponding property on <see cref="EcsDocument"/>
+		/// <para></para>
+		/// <para>See <see cref="LogTemplateProperties"/> for a strongly typed list of valid ECS log template properties</para>
+		/// <para>If its not a supported ECS log template property or using the wrong type:</para>
+		/// <list type="bullet">
+		/// <item>Assigns strings to <see cref="EcsDocument.Labels"/> on <see cref="EcsDocument"/></item>
+		/// <item>Assigns everything else to <see cref="EcsDocument.Metadata"/> on <see cref="EcsDocument"/></item>
+		/// </list>
+		/// </summary>
+		/// <para@(Raw("m")) name="path">Either a supported ECS Log Template property or any key</para@(Raw("m"))>
+		/// <para@(Raw("m")) name="value">The value to persist</para@(Raw("m"))>
+		public void SetLogMessageProperty(string path, object value)
+		{
+			var assigned = LogTemplateProperties.All.Contains(path) && TrySet(this, path, value);
+			if (!assigned) 
+				SetMetaOrLabel(this, path, value);
+		}
+	}
+	internal static partial class PropDispatch
+	{
+		public static bool TrySet(EcsDocument document, string path, object value) 
+		{
+			switch (path) 
+			{
+	@foreach (var prop in Model.Base.BaseFieldSet.SettableProperties)
+	{
+<text>				case "@prop.FullPath":
+				case "@prop.LogTemplateAlternative":
+</text>
+	}
+					return TrySet@(@Model.Base.Name)(document, path, value);
+@foreach (var entity in Model.EntityClasses)
+{
+	if (!entity.BaseFieldSet.SettableProperties.Any())
+	{
+		continue;
+	}
+	@foreach (var prop in entity.BaseFieldSet.SettableProperties) 
+	{
+<text>				case "@prop.FullPath":
+				case "@prop.LogTemplateAlternative":
+</text>
+	}
+<text>					return TrySet@(@entity.Name)(document, path, value);
+</text>
+}
+				default:
+					SetMetaOrLabel(document, path, value);
+					return true;
+			}
+		}
+
+		public static bool TrySet@(@Model.Base.Name)(EcsDocument document, string path, object value)
+		{
+			Func@(Raw("<"))@(Model.Base.Name), object, bool@(Raw(">")) assign = path switch
+			{
+		@foreach (var prop in Model.Base.BaseFieldSet.SettableProperties)
+		{
+<text>				"@prop.FullPath" => static (e, v) => @(prop.CastFromObject)(e, v, static (ee, p) => ee.@(prop.Name) = p),
+				"@prop.LogTemplateAlternative" => static (e, v) => @(prop.CastFromObject)(e, v, static (ee, p) => ee.@(prop.Name) = p),
+</text>
+		}
+				_ => null
+			};
+			return assign != null && assign(document, value);
+		}
+@foreach (var entity in Model.EntityClasses)
+{
+<text>
+		public static bool TrySet@(entity.Name)(EcsDocument document, string path, object value)
+		{
+			Func@(Raw("<"))@(entity.Name), object, bool@(Raw(">")) assign = path switch
+			{
+		@foreach (var prop in entity.BaseFieldSet.SettableProperties)
+		{
+<text>				"@prop.FullPath" => static (e, v) => @(prop.CastFromObject)(e, v, static (ee, p) => ee.@(prop.Name) = p),
+				"@prop.LogTemplateAlternative" => static (e, v) => @(prop.CastFromObject)(e, v, static (ee, p) => ee.@(prop.Name) = p),
+</text>
+		}
+				_ => null
+			};
+			if (assign == null) return false;
+
+			var entity = document.@(entity.Name) ?? new @(entity.Name)();
+			var assigned = assign(entity, value);
+			if (assigned) document.@(entity.Name) = entity;
+			return assigned;
+		}
+</text>
+}
+	}
+}

src/Elastic.CommonSchema.Log4net/LoggingEventConverter.cs

@@ -13,7 +13,8 @@ namespace Elastic.CommonSchema.Log4net;
 internal static class LoggingEventConverter
 {
 	public static EcsDocument ToEcs(this LoggingEvent loggingEvent)
-		=> new()
+	{
+		var ecsDocument = new EcsDocument()
 		{
 			Timestamp = loggingEvent.TimeStamp,
 			Ecs = new Ecs { Version = EcsDocument.Version },
@@ -23,8 +24,14 @@ public static EcsDocument ToEcs(this LoggingEvent loggingEvent)
 			Error = GetError(loggingEvent),
 			Process = GetProcess(loggingEvent),
 			Host = GetHost(loggingEvent),
-			Metadata = GetMetadata(loggingEvent)
 		};
+		var metadata = GetMetadata(loggingEvent);
+		if (metadata == null) return ecsDocument;
+
+		foreach(var kv in metadata)
+			ecsDocument.SetLogMessageProperty(kv.Key, kv.Value);
+		return ecsDocument;
+	}
 
 	private static Log GetLog(LoggingEvent loggingEvent)
 	{
@@ -134,9 +141,7 @@ private static MetadataDictionary GetMetadata(LoggingEvent loggingEvent)
 	{
 		var properties = loggingEvent.GetProperties();
 		if (properties.Count == 0)
-		{
 			return null;
-		}
 
 		var metadata = new MetadataDictionary();
 

src/Elastic.CommonSchema.Log4net/README.md

@@ -36,6 +36,11 @@ The `Layout = new EcsLayout()` line then instructs log4net to use ECS layout.
 The sample above uses the console appender, but you are free to use any appender of your choice, perhaps consider using a
 filesystem target and [Elastic Filebeat](https://www.elastic.co/downloads/beats/filebeat) for durable and reliable ingestion.
 
+### ECS Aware Properties
+
+Any valid ECS log template properties that is available under `LogTemplateProperties.*` e.g `LogTemplateProperties.TraceId`
+is supported and will directly set the appropriate ECS field.
+
 ## Output
 
 Apart from [mandatory fields](https://www.elastic.co/guide/en/ecs/current/ecs-guidelines.html#_general_guidelines), the output contains additional data:

src/Elastic.CommonSchema.NLog/EcsLayout.cs

@@ -192,7 +192,6 @@ protected override void RenderFormattedMessage(LogEventInfo logEvent, StringBuil
 				Log = GetLog(logEvent),
 				Service = GetService(logEvent),
 				Event = GetEvent(logEvent),
-				Metadata = GetMetadata(logEvent),
 				Process = GetProcess(logEvent),
 				TraceId = GetTrace(logEvent),
 				TransactionId = GetTransaction(logEvent),
@@ -206,6 +205,9 @@ protected override void RenderFormattedMessage(LogEventInfo logEvent, StringBuil
 				Http = GetHttp(logEvent),
 				Url = GetUrl(logEvent),
 			};
+			var metadata = GetMetadata(logEvent) ?? MetadataDictionary.Default;
+			foreach(var kv in metadata)
+				ecsEvent.SetLogMessageProperty(kv.Key, kv.Value);
 
 			//Give any deriving classes a chance to enrich the event
 			EnrichEvent(logEvent, ref ecsEvent);