diff --git a/global.json b/global.json index 341f9fea..99e7444e 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "2.1.4" + "version": "2.1.500" } } \ No newline at end of file diff --git a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj index b1d79191..d2e3559f 100644 --- a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj +++ b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj @@ -33,7 +33,6 @@ - @@ -43,7 +42,16 @@ + + + + + + + + + 1591;1701;1702 diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs index 0bb75390..d89136fd 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs @@ -71,7 +71,7 @@ protected override void EmitBatch(IEnumerable events) var items = result.Body["items"]; foreach (var item in items) { - if (item.index != null && item.index.error != null) + if (item["index"] != null && item["index"]["error"] != null) { var e = events.ElementAt(indexer); if (_state.Options.EmitEventFailure.HasFlag(EmitEventFailureHandling.WriteToSelfLog)) @@ -80,8 +80,8 @@ protected override void EmitBatch(IEnumerable events) SelfLog.WriteLine( "Failed to store event with template '{0}' into Elasticsearch. Elasticsearch reports for index {1} the following: {2}", e.MessageTemplate, - item.index._index, - item.index.error); + item["index"]["_index"], + _state.Serialize(item["index"]["error"])); } if (_state.Options.EmitEventFailure.HasFlag(EmitEventFailureHandling.WriteToFailureSink) && diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs index 3847a4b0..0325992b 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs @@ -19,7 +19,11 @@ public enum AutoRegisterTemplateVersion /// /// Elasticsearch version >= version 6.0 /// - ESv6 = 2 + ESv6 = 2, + /// + /// Elasticsearch version >= version 7.0 + /// + ESv7 = 3 } /// @@ -45,6 +49,8 @@ public static object GetTemplate( return GetTemplateESv5(settings, templateMatchString); case AutoRegisterTemplateVersion.ESv6: return GetTemplateESv6(settings, templateMatchString); + case AutoRegisterTemplateVersion.ESv7: + return GetTemplateESv7(settings, templateMatchString); case AutoRegisterTemplateVersion.ESv2: return GetTemplateESv2(settings, templateMatchString); default: @@ -52,6 +58,96 @@ public static object GetTemplate( } } + private static object GetTemplateESv7( + Dictionary settings, + string templateMatchString) + { + return new + { + index_patterns = new[] { templateMatchString }, + settings = settings, + mappings = new + { + dynamic_templates = new List + { + //when you use serilog as an adaptor for third party frameworks + //where you have no control over the log message they typically + //contain {0} ad infinitum, we force numeric property names to + //contain strings by default. + { + new + { + numerics_in_fields = new + { + path_match = @"fields\.[\d+]$", + match_pattern = "regex", + mapping = new + { + type = "text", + index = true, + norms = false + } + } + } + }, + { + new + { + string_fields = new + { + match = "*", + match_mapping_type = "string", + mapping = new + { + type = "text", + index = true, + norms = false, + fields = new + { + raw = new + { + type = "keyword", + index = true, + ignore_above = 256 + } + } + } + } + } + } + }, + properties = new Dictionary + { + {"message", new {type = "text", index = "true"}}, + { + "exceptions", new + { + type = "nested", + properties = new Dictionary + { + {"Depth", new {type = "integer"}}, + {"RemoteStackIndex", new {type = "integer"}}, + {"HResult", new {type = "integer"}}, + {"StackTraceString", new {type = "text", index = "true"}}, + {"RemoteStackTraceString", new {type = "text", index = "true"}}, + { + "ExceptionMessage", new + { + type = "object", + properties = new Dictionary + { + {"MemberType", new {type = "integer"}}, + } + } + } + } + } + } + } + } + }; + } + private static object GetTemplateESv6( Dictionary settings, string templateMatchString) diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs index af1a6cbe..679857c9 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs @@ -59,9 +59,18 @@ public static ElasticsearchSinkState Create(ElasticsearchSinkOptions options) private ElasticsearchSinkState(ElasticsearchSinkOptions options) { if (string.IsNullOrWhiteSpace(options.IndexFormat)) throw new ArgumentException("options.IndexFormat"); - if (string.IsNullOrWhiteSpace(options.TypeName)) throw new ArgumentException("options.TypeName"); if (string.IsNullOrWhiteSpace(options.TemplateName)) throw new ArgumentException("options.TemplateName"); + // Strip type argument if ESv7 since multiple types are not supported anymore + if (options.AutoRegisterTemplateVersion == AutoRegisterTemplateVersion.ESv7) + { + options.TypeName = "_doc"; + } + else + { + if (string.IsNullOrWhiteSpace(options.TypeName)) throw new ArgumentException("options.TypeName"); + } + _templateName = options.TemplateName; _templateMatchString = IndexFormatRegex.Replace(options.IndexFormat, @"$1*$2"); @@ -70,6 +79,7 @@ private ElasticsearchSinkState(ElasticsearchSinkOptions options) _options = options; + var configuration = new ConnectionConfiguration(options.ConnectionPool, options.Connection, options.Serializer) .RequestTimeout(options.ConnectionTimeout); @@ -112,7 +122,7 @@ public static ITextFormatter CreateDefaultDurableFormatter(ElasticsearchSinkOpti public string Serialize(object o) { - return _client.Serializer.SerializeToString(o, SerializationFormatting.None); + return _client.Serializer.SerializeToString(o, formatting: SerializationFormatting.None); } public string GetIndexForEvent(LogEvent e, DateTimeOffset offset) @@ -149,6 +159,7 @@ public void RegisterTemplateIfNeeded() } } + Console.WriteLine(_client.Serializer.SerializeToString(GetTemplateData())); var result = _client.IndicesPutTemplateForAll(_templateName, GetTempatePostData()); if (!result.Success) diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/SerializerAdapter.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/SerializerAdapter.cs index c551df02..c171a79b 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/SerializerAdapter.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/SerializerAdapter.cs @@ -53,7 +53,7 @@ public Task SerializeAsync(T data, Stream stream, public string SerializeToString(object value) { - return _elasticsearchSerializer.SerializeToString(value, SerializationFormatting.None); + return _elasticsearchSerializer.SerializeToString(value, formatting: SerializationFormatting.None); } } } \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj b/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj index 03f8600f..ed9333dd 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj @@ -24,9 +24,11 @@ + + @@ -39,7 +41,6 @@ - @@ -61,8 +62,19 @@ + + + + + + + + + + + $(DefineConstants);DOTNETCORE;NO_SERIALIZATION diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv7TemplateTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv7TemplateTests.cs new file mode 100644 index 00000000..6180ed44 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv7TemplateTests.cs @@ -0,0 +1,66 @@ +using System; +using System.Reflection; +using FluentAssertions; +using Newtonsoft.Json.Linq; +using Xunit; + +namespace Serilog.Sinks.Elasticsearch.Tests.Templating +{ + public class Sendsv7TemplateTests : ElasticsearchSinkTestsBase + { + private readonly Tuple _templatePut; + + public Sendsv7TemplateTests() + { + _options.AutoRegisterTemplate = true; + _options.AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7; + + var loggerConfig = new LoggerConfiguration() + .MinimumLevel.Debug() + .Enrich.WithMachineName() + .WriteTo.ColoredConsole() + .WriteTo.Elasticsearch(_options); + + var logger = loggerConfig.CreateLogger(); + using (logger as IDisposable) + { + logger.Error("Test exception. Should not contain an embedded exception object."); + } + + this._seenHttpPosts.Should().NotBeNullOrEmpty().And.HaveCount(1); + this._seenHttpPuts.Should().NotBeNullOrEmpty().And.HaveCount(1); + _templatePut = this._seenHttpPuts[0]; + } + + [Fact] + public void ShouldRegisterTheCorrectTemplateOnRegistration() + { + + var method = typeof(Sendsv7TemplateTests).GetMethod(nameof(ShouldRegisterTheCorrectTemplateOnRegistration)); + JsonEquals(_templatePut.Item2, method, "template_v7.json"); + } + + [Fact] + public void TemplatePutToCorrectUrl() + { + var uri = _templatePut.Item1; + uri.AbsolutePath.Should().Be("/_template/serilog-events-template"); + } + + protected void JsonEquals(string json, MethodBase method, string fileName = null) + { +#if DOTNETCORE + var assembly = typeof(Sendsv7TemplateTests).GetTypeInfo().Assembly; +#else + var assembly = Assembly.GetExecutingAssembly(); +#endif + var expected = TestDataHelper.ReadEmbeddedResource(assembly, fileName ?? "template.json"); + + var nJson = JObject.Parse(json); + var nOtherJson = JObject.Parse(expected); + var equals = JToken.DeepEquals(nJson, nOtherJson); + if (equals) return; + expected.Should().BeEquivalentTo(json); + } + } +} \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v7.json b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v7.json new file mode 100644 index 00000000..585f5323 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v7.json @@ -0,0 +1,75 @@ +{ + "index_patterns": [ "logstash-*" ], + "settings": { + "index.refresh_interval": "5s" + }, + "mappings": { + "dynamic_templates": [ + { + "numerics_in_fields": { + "path_match": "fields\\.[\\d+]$", + "match_pattern": "regex", + "mapping": { + "type": "text", + "index": true, + "norms": false + } + } + }, + { + "string_fields": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "text", + "index": true, + "norms": false, + "fields": { + "raw": { + "type": "keyword", + "index": true, + "ignore_above": 256 + } + } + } + } + } + ], + "properties": { + "message": { + "type": "text", + "index": "true" + }, + "exceptions": { + "type": "nested", + "properties": { + "Depth": { + "type": "integer" + }, + "RemoteStackIndex": { + "type": "integer" + }, + "HResult": { + "type": "integer" + }, + "StackTraceString": { + "type": "text", + "index": "true" + }, + "RemoteStackTraceString": { + "type": "text", + "index": "true" + }, + "ExceptionMessage": { + "type": "object", + "properties": { + "MemberType": { + "type": "integer" + } + } + } + } + } + } + } +}