Elasticsearch.Net 7.0.0 support, V7 templates

global.json

@@ -1,5 +1,5 @@
 {
     "sdk": {
-        "version": "2.1.4"
+        "version": "2.1.500"
     }
 }

src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj

@@ -33,7 +33,6 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Elasticsearch.Net" Version="6.0.0" />
     <PackageReference Include="Serilog" Version="2.8.0" />
     <PackageReference Include="Serilog.Formatting.Compact" Version="1.0.0" />
     <PackageReference Include="Serilog.Sinks.File" Version="4.0.0" />
@@ -43,7 +42,16 @@
   <ItemGroup Condition=" '$(TargetFramework)' == 'net45' ">
     <Reference Include="System" />
     <Reference Include="Microsoft.CSharp" />
+    <PackageReference Include="Elasticsearch.Net" Version="6.0.0" />
   </ItemGroup>
+    <ItemGroup Condition=" '$(TargetFramework)' == 'netstandard1.3' ">
+        <PackageReference Include="Elasticsearch.Net" Version="6.0.0" />
+    </ItemGroup>
+
+    <ItemGroup Condition=" '$(TargetFramework)' == 'netstandard2.0' ">
+        <PackageReference Include="Elasticsearch.Net" Version="7.0.0-alpha2" />
+        <PackageReference Include="Microsoft.CSharp" Version="4.5.0" />
+    </ItemGroup>
 
   <PropertyGroup Condition=" '$(TargetFramework)' == 'netstandard1.3' ">
     <NoWarn>1591;1701;1702</NoWarn>

src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs

@@ -71,7 +71,7 @@ protected override void EmitBatch(IEnumerable<LogEvent> events)
                 var items = result.Body["items"];
                 foreach (var item in items)
                 {
-                    if (item.index != null && item.index.error != null)
+                    if (item["index"] != null && item["index"]["error"] != null)
                     {
                         var e = events.ElementAt(indexer);
                         if (_state.Options.EmitEventFailure.HasFlag(EmitEventFailureHandling.WriteToSelfLog))
@@ -80,8 +80,8 @@ protected override void EmitBatch(IEnumerable<LogEvent> events)
                             SelfLog.WriteLine(
                                 "Failed to store event with template '{0}' into Elasticsearch. Elasticsearch reports for index {1} the following: {2}",
                                 e.MessageTemplate,
-                                item.index._index,
-                                item.index.error);
+                                item["index"]["_index"],
+                                _state.Serialize(item["index"]["error"]));
                         }
 
                         if (_state.Options.EmitEventFailure.HasFlag(EmitEventFailureHandling.WriteToFailureSink) &&

src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs

@@ -19,7 +19,11 @@ public enum AutoRegisterTemplateVersion
         /// <summary>
         /// Elasticsearch version &gt;= version 6.0
         /// </summary>
-        ESv6 = 2
+        ESv6 = 2,
+        /// <summary>
+        /// Elasticsearch version &gt;= version 7.0
+        /// </summary>
+        ESv7 = 3
     }
 
     /// <summary>
@@ -45,13 +49,105 @@ public static object GetTemplate(
                     return GetTemplateESv5(settings, templateMatchString);
                 case AutoRegisterTemplateVersion.ESv6:
                     return GetTemplateESv6(settings, templateMatchString);
+                case AutoRegisterTemplateVersion.ESv7:
+                    return GetTemplateESv7(settings, templateMatchString);
                 case AutoRegisterTemplateVersion.ESv2:
                     return GetTemplateESv2(settings, templateMatchString);
                 default:
                     throw new ArgumentOutOfRangeException(nameof(version), version, null);
             }
         }
 
+        private static object GetTemplateESv7(
+            Dictionary<string, string> settings,
+            string templateMatchString)
+        {
+            return new
+            {
+                index_patterns = new[] { templateMatchString },
+                settings = settings,
+                mappings = new
+                {
+                    dynamic_templates = new List<Object>
+                    {
+                        //when you use serilog as an adaptor for third party frameworks
+                        //where you have no control over the log message they typically
+                        //contain {0} ad infinitum, we force numeric property names to
+                        //contain strings by default.
+                        {
+                            new
+                            {
+                                numerics_in_fields = new
+                                {
+                                    path_match = @"fields\.[\d+]$",
+                                    match_pattern = "regex",
+                                    mapping = new
+                                    {
+                                        type = "text",
+                                        index = true,
+                                        norms = false
+                                    }
+                                }
+                            }
+                        },
+                        {
+                            new
+                            {
+                                string_fields = new
+                                {
+                                    match = "*",
+                                    match_mapping_type = "string",
+                                    mapping = new
+                                    {
+                                        type = "text",
+                                        index = true,
+                                        norms = false,
+                                        fields = new
+                                        {
+                                            raw = new
+                                            {
+                                                type = "keyword",
+                                                index = true,
+                                                ignore_above = 256
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    },
+                    properties = new Dictionary<string, object>
+                    {
+                        {"message", new {type = "text", index = "true"}},
+                        {
+                            "exceptions", new
+                            {
+                                type = "nested",
+                                properties = new Dictionary<string, object>
+                                {
+                                    {"Depth", new {type = "integer"}},
+                                    {"RemoteStackIndex", new {type = "integer"}},
+                                    {"HResult", new {type = "integer"}},
+                                    {"StackTraceString", new {type = "text", index = "true"}},
+                                    {"RemoteStackTraceString", new {type = "text", index = "true"}},
+                                    {
+                                        "ExceptionMessage", new
+                                        {
+                                            type = "object",
+                                            properties = new Dictionary<string, object>
+                                            {
+                                                {"MemberType", new {type = "integer"}},
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            };
+        }
+
         private static object GetTemplateESv6(
             Dictionary<string, string> settings,
             string templateMatchString)

src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs

@@ -59,9 +59,18 @@ public static ElasticsearchSinkState Create(ElasticsearchSinkOptions options)
         private ElasticsearchSinkState(ElasticsearchSinkOptions options)
         {
             if (string.IsNullOrWhiteSpace(options.IndexFormat)) throw new ArgumentException("options.IndexFormat");
-            if (string.IsNullOrWhiteSpace(options.TypeName)) throw new ArgumentException("options.TypeName");
             if (string.IsNullOrWhiteSpace(options.TemplateName)) throw new ArgumentException("options.TemplateName");
 
+            // Strip type argument if ESv7 since multiple types are not supported anymore
+            if (options.AutoRegisterTemplateVersion == AutoRegisterTemplateVersion.ESv7)
+            {
+                options.TypeName = "_doc";
+            }
+            else
+            {
+                if (string.IsNullOrWhiteSpace(options.TypeName)) throw new ArgumentException("options.TypeName");
+            }
+
             _templateName = options.TemplateName;
             _templateMatchString = IndexFormatRegex.Replace(options.IndexFormat, @"$1*$2");
 
@@ -70,6 +79,7 @@ private ElasticsearchSinkState(ElasticsearchSinkOptions options)
 
             _options = options;
 
+
             var configuration = new ConnectionConfiguration(options.ConnectionPool, options.Connection, options.Serializer)
                 .RequestTimeout(options.ConnectionTimeout);
 
@@ -112,7 +122,7 @@ public static ITextFormatter CreateDefaultDurableFormatter(ElasticsearchSinkOpti
 
         public string Serialize(object o)
         {
-            return _client.Serializer.SerializeToString(o, SerializationFormatting.None);
+            return _client.Serializer.SerializeToString(o, formatting: SerializationFormatting.None);
         }
 
         public string GetIndexForEvent(LogEvent e, DateTimeOffset offset)
@@ -149,6 +159,7 @@ public void RegisterTemplateIfNeeded()
                     }
                 }
 
+                Console.WriteLine(_client.Serializer.SerializeToString(GetTemplateData()));
                 var result = _client.IndicesPutTemplateForAll<DynamicResponse>(_templateName, GetTempatePostData());
 
                 if (!result.Success)

src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/SerializerAdapter.cs

@@ -53,7 +53,7 @@ public Task SerializeAsync<T>(T data, Stream stream,
 
          public string SerializeToString(object value)
         {
-            return _elasticsearchSerializer.SerializeToString(value, SerializationFormatting.None);
+            return _elasticsearchSerializer.SerializeToString(value, formatting: SerializationFormatting.None);
         }
     }
 } 

test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj

@@ -24,9 +24,11 @@
     <None Remove="Templating\template_v2.json" />
     <None Remove="Templating\template_v5.json" />
     <None Remove="Templating\template_v6.json" />
+    <None Remove="Templating\template_v7.json" />
   </ItemGroup>
 
   <ItemGroup>
+    <EmbeddedResource Include="Templating\template_v7.json" />
     <EmbeddedResource Include="Templating\template_v6.json" />
     <EmbeddedResource Include="Templating\template_v5.json" />
     <EmbeddedResource Include="Templating\template_v2.json" />
@@ -39,7 +41,6 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Elasticsearch.Net" Version="6.0.0" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.5.0" />
     <PackageReference Include="Microsoft.NETCore.Platforms" Version="2.0.1" />
     <PackageReference Include="NEST.JsonNetSerializer" Version="6.0.0" />
@@ -61,8 +62,19 @@
     <Reference Include="System.Threading.Tasks" />
     <Reference Include="System" />
     <Reference Include="Microsoft.CSharp" />
+    <PackageReference Include="Elasticsearch.Net" Version="6.0.0" />
   </ItemGroup>
 
+
+    <ItemGroup Condition=" '$(TargetFramework)' == 'netstandard1.3' ">
+        <PackageReference Include="Elasticsearch.Net" Version="6.0.0" />
+    </ItemGroup>
+
+    <ItemGroup Condition=" '$(TargetFramework)' == 'netstandard2.0' ">
+        <PackageReference Include="Elasticsearch.Net" Version="7.0.0-alpha2" />
+        <PackageReference Include="Microsoft.CSharp" Version="4.5.0" />
+    </ItemGroup>
+
   <PropertyGroup Condition=" '$(TargetFramework)' == 'netcoreapp1.1' ">
     <DefineConstants>$(DefineConstants);DOTNETCORE;NO_SERIALIZATION</DefineConstants>
   </PropertyGroup>

test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv7TemplateTests.cs

@@ -0,0 +1,66 @@
+using System;
+using System.Reflection;
+using FluentAssertions;
+using Newtonsoft.Json.Linq;
+using Xunit;
+
+namespace Serilog.Sinks.Elasticsearch.Tests.Templating
+{
+    public class Sendsv7TemplateTests : ElasticsearchSinkTestsBase
+    {
+        private readonly Tuple<Uri, string> _templatePut;
+
+        public Sendsv7TemplateTests()
+        {
+            _options.AutoRegisterTemplate = true;
+            _options.AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7;
+
+            var loggerConfig = new LoggerConfiguration()
+                .MinimumLevel.Debug()
+                .Enrich.WithMachineName()
+                .WriteTo.ColoredConsole()
+                .WriteTo.Elasticsearch(_options);
+
+            var logger = loggerConfig.CreateLogger();
+            using (logger as IDisposable)
+            {
+                logger.Error("Test exception. Should not contain an embedded exception object.");
+            }
+
+            this._seenHttpPosts.Should().NotBeNullOrEmpty().And.HaveCount(1);
+            this._seenHttpPuts.Should().NotBeNullOrEmpty().And.HaveCount(1);
+            _templatePut = this._seenHttpPuts[0];
+        }
+
+        [Fact]
+        public void ShouldRegisterTheCorrectTemplateOnRegistration()
+        {
+
+            var method = typeof(Sendsv7TemplateTests).GetMethod(nameof(ShouldRegisterTheCorrectTemplateOnRegistration));
+            JsonEquals(_templatePut.Item2, method, "template_v7.json");
+        }
+
+        [Fact]
+        public void TemplatePutToCorrectUrl()
+        {
+            var uri = _templatePut.Item1;
+            uri.AbsolutePath.Should().Be("/_template/serilog-events-template");
+        }
+
+        protected void JsonEquals(string json, MethodBase method, string fileName = null)
+        {
+#if DOTNETCORE
+            var assembly = typeof(Sendsv7TemplateTests).GetTypeInfo().Assembly;
+#else
+            var assembly = Assembly.GetExecutingAssembly();
+#endif
+            var expected = TestDataHelper.ReadEmbeddedResource(assembly, fileName ?? "template.json");
+
+            var nJson = JObject.Parse(json);
+            var nOtherJson = JObject.Parse(expected);
+            var equals = JToken.DeepEquals(nJson, nOtherJson);
+            if (equals) return;
+            expected.Should().BeEquivalentTo(json);
+        }
+    }
+}