From 1dc059ddde154702dbc9051047e6d36b126b5be0 Mon Sep 17 00:00:00 2001 From: Nikolaj Dam Larsen Date: Fri, 22 Nov 2019 07:54:57 +0100 Subject: [PATCH 1/6] Change default TypeName to '_doc' --- .../Sinks/ElasticSearch/ElasticsearchSinkOptions.cs | 2 +- .../ElasticsearchSinkTestsBase.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs index 0c450796..f96037b4 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs @@ -278,7 +278,7 @@ public ElasticsearchSinkOptions() /// The default Elasticsearch type name used for Elasticsearch versions prior to 7. /// As of Elasticsearch 7 and up _type has been removed. /// - public static string DefaultTypeName { get; } = "logevent"; + public static string DefaultTypeName { get; } = "_doc"; /// /// Instructs the sink to auto detect the running Elasticsearch version. diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs b/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs index e42f11c3..8eca8337 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs @@ -73,7 +73,7 @@ protected IList GetPostedLogEvents(int expectedCount) } action.IndexAction.Should().NotBeNull(); action.IndexAction.Index.Should().NotBeNullOrEmpty().And.StartWith("logstash-"); - action.IndexAction.Type.Should().NotBeNullOrEmpty().And.Be("logevent"); + action.IndexAction.Type.Should().NotBeNullOrEmpty().And.Be("_doc"); SerilogElasticsearchEvent actionMetaData; try From 11793235e8b3147af342acbc9553d700f11e5d36 Mon Sep 17 00:00:00 2001 From: Michiel van Oudheusden Date: Mon, 16 Dec 2019 10:52:39 +0100 Subject: [PATCH 2/6] Added tests for 0 replicas --- .../ElasticsearchSinkTestsBase.cs | 5 +- .../Serilog.Sinks.Elasticsearch.Tests.csproj | 4 + .../DoNotRegisterTemplateIfItExists.cs | 2 +- .../SetFiveReplicasInTemplateTests.cs | 65 +++++++++++++++ .../SetZeroReplicasInTemplateTests.cs | 65 +++++++++++++++ .../Templating/template_0replicas.json | 82 +++++++++++++++++++ .../Templating/template_5replicas.json | 82 +++++++++++++++++++ 7 files changed, 302 insertions(+), 3 deletions(-) create mode 100644 test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetFiveReplicasInTemplateTests.cs create mode 100644 test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetZeroReplicasInTemplateTests.cs create mode 100644 test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_0replicas.json create mode 100644 test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_5replicas.json diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs b/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs index e42f11c3..a2cead27 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs @@ -69,7 +69,7 @@ protected IList GetPostedLogEvents(int expectedCount) } catch (Exception e) { - throw new Exception(string.Format("Can not deserialize into BulkOperation \r\n:{0}", totalBulks[i]), e); + throw new Exception($"Can not deserialize into BulkOperation \r\n:{totalBulks[i]}", e); } action.IndexAction.Should().NotBeNull(); action.IndexAction.Index.Should().NotBeNullOrEmpty().And.StartWith("logstash-"); @@ -82,7 +82,8 @@ protected IList GetPostedLogEvents(int expectedCount) } catch (Exception e) { - throw new Exception(string.Format("Can not deserialize into SerilogElasticsearchMessage \r\n:{0}", totalBulks[i + 1]), e); + throw new Exception( + $"Can not deserialize into SerilogElasticsearchMessage \r\n:{totalBulks[i + 1]}", e); } actionMetaData.Should().NotBeNull(); bulkActions.Add(actionMetaData); diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj b/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj index a1f18554..8874a3f9 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj @@ -24,6 +24,8 @@ + + @@ -31,6 +33,8 @@ + + diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/DoNotRegisterTemplateIfItExists.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/DoNotRegisterTemplateIfItExists.cs index 3ac15a29..96300ab6 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/DoNotRegisterTemplateIfItExists.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/DoNotRegisterTemplateIfItExists.cs @@ -25,7 +25,7 @@ private void DoRegister() } [Fact] - public void WhenTempplateExists_ShoudNotSendAPutTemplate() + public void WhenTemplateExists_ShouldNotSendAPutTemplate() { DoRegister(); this._seenHttpPosts.Should().NotBeNullOrEmpty().And.HaveCount(1); diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetFiveReplicasInTemplateTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetFiveReplicasInTemplateTests.cs new file mode 100644 index 00000000..602e3377 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetFiveReplicasInTemplateTests.cs @@ -0,0 +1,65 @@ +using System; +using System.Reflection; +using FluentAssertions; +using Newtonsoft.Json.Linq; +using Xunit; + +namespace Serilog.Sinks.Elasticsearch.Tests.Templating +{ + public class SetFiveReplicasInTemplateTests : ElasticsearchSinkTestsBase + { + private readonly Tuple _templatePut; + + public SetFiveReplicasInTemplateTests() + { + _options.AutoRegisterTemplate = true; + _options.NumberOfReplicas = 5; + + var loggerConfig = new LoggerConfiguration() + .MinimumLevel.Debug() + .Enrich.WithMachineName() + .WriteTo.ColoredConsole() + .WriteTo.Elasticsearch(_options); + + var logger = loggerConfig.CreateLogger(); + using (logger as IDisposable) + { + logger.Error("Test exception. Should not contain an embedded exception object."); + } + + this._seenHttpPosts.Should().NotBeNullOrEmpty().And.HaveCount(1); + this._seenHttpPuts.Should().NotBeNullOrEmpty().And.HaveCount(1); + _templatePut = this._seenHttpPuts[0]; + } + + [Fact] + public void ShouldRegisterTheCorrectTemplateOnRegistration() + { + var method = typeof(SendsTemplateTests).GetMethod(nameof(ShouldRegisterTheCorrectTemplateOnRegistration)); + JsonEquals(_templatePut.Item2, method, "template"); + } + + [Fact] + public void TemplatePutToCorrectUrl() + { + var uri = _templatePut.Item1; + uri.AbsolutePath.Should().Be("/_template/serilog-events-template"); + } + + protected void JsonEquals(string json, MethodBase method, string fileName = null) + { +#if DOTNETCORE + var assembly = typeof(SendsTemplateTests).GetTypeInfo().Assembly; +#else + var assembly = Assembly.GetExecutingAssembly(); +#endif + var expected = TestDataHelper.ReadEmbeddedResource(assembly, "template_5replicas.json"); + + var nJson = JObject.Parse(json); + var nOtherJson = JObject.Parse(expected); + var equals = JToken.DeepEquals(nJson, nOtherJson); + if (equals) return; + expected.Should().BeEquivalentTo(json); + } + } +} \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetZeroReplicasInTemplateTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetZeroReplicasInTemplateTests.cs new file mode 100644 index 00000000..47c87812 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetZeroReplicasInTemplateTests.cs @@ -0,0 +1,65 @@ +using System; +using System.Reflection; +using FluentAssertions; +using Newtonsoft.Json.Linq; +using Xunit; + +namespace Serilog.Sinks.Elasticsearch.Tests.Templating +{ + public class SetZeroReplicasInTemplateTests : ElasticsearchSinkTestsBase + { + private readonly Tuple _templatePut; + + public SetZeroReplicasInTemplateTests() + { + _options.AutoRegisterTemplate = true; + _options.NumberOfReplicas = 0; + + var loggerConfig = new LoggerConfiguration() + .MinimumLevel.Debug() + .Enrich.WithMachineName() + .WriteTo.ColoredConsole() + .WriteTo.Elasticsearch(_options); + + var logger = loggerConfig.CreateLogger(); + using (logger as IDisposable) + { + logger.Error("Test exception. Should not contain an embedded exception object."); + } + + this._seenHttpPosts.Should().NotBeNullOrEmpty().And.HaveCount(1); + this._seenHttpPuts.Should().NotBeNullOrEmpty().And.HaveCount(1); + _templatePut = this._seenHttpPuts[0]; + } + + [Fact] + public void ShouldRegisterTheCorrectTemplateOnRegistration() + { + var method = typeof(SendsTemplateTests).GetMethod(nameof(ShouldRegisterTheCorrectTemplateOnRegistration)); + JsonEquals(_templatePut.Item2, method, "template"); + } + + [Fact] + public void TemplatePutToCorrectUrl() + { + var uri = _templatePut.Item1; + uri.AbsolutePath.Should().Be("/_template/serilog-events-template"); + } + + protected void JsonEquals(string json, MethodBase method, string fileName = null) + { +#if DOTNETCORE + var assembly = typeof(SendsTemplateTests).GetTypeInfo().Assembly; +#else + var assembly = Assembly.GetExecutingAssembly(); +#endif + var expected = TestDataHelper.ReadEmbeddedResource(assembly, "template_0replicas.json"); + + var nJson = JObject.Parse(json); + var nOtherJson = JObject.Parse(expected); + var equals = JToken.DeepEquals(nJson, nOtherJson); + if (equals) return; + expected.Should().BeEquivalentTo(json); + } + } +} \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_0replicas.json b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_0replicas.json new file mode 100644 index 00000000..82b465b4 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_0replicas.json @@ -0,0 +1,82 @@ +{ + "template": "logstash-*", + "settings": { + "index.refresh_interval": "5s", + "number_of_replicas": "0" + }, + "mappings": { + "_default_": { + "_all": { + "enabled": true, + "omit_norms" : true + }, + "dynamic_templates": [ + { + "numerics_in_fields": { + "path_match":"fields\\.[\\d+]$", + "match_pattern":"regex", + "mapping": { + "type":"string", + "index":"analyzed", + "omit_norms":true + } + } + }, + { + "string_fields": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "string", + "index": "analyzed", + "omit_norms": true, + "fields": { + "raw": { + "type": "string", + "index": "not_analyzed", + "ignore_above": 256 + } + } + } + } + } + ], + "properties": { + "message": { + "type": "string", + "index": "analyzed" + }, + "exceptions": { + "type": "nested", + "properties": { + "Depth": { + "type": "integer" + }, + "RemoteStackIndex": { + "type": "integer" + }, + "HResult": { + "type": "integer" + }, + "StackTraceString": { + "type": "string", + "index": "analyzed" + }, + "RemoteStackTraceString": { + "type": "string", + "index": "analyzed" + }, + "ExceptionMessage": { + "type": "object", + "properties": { + "MemberType": { + "type": "integer" + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_5replicas.json b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_5replicas.json new file mode 100644 index 00000000..50c4035f --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_5replicas.json @@ -0,0 +1,82 @@ +{ + "template": "logstash-*", + "settings": { + "index.refresh_interval": "5s", + "number_of_replicas": "5" + }, + "mappings": { + "_default_": { + "_all": { + "enabled": true, + "omit_norms" : true + }, + "dynamic_templates": [ + { + "numerics_in_fields": { + "path_match":"fields\\.[\\d+]$", + "match_pattern":"regex", + "mapping": { + "type":"string", + "index":"analyzed", + "omit_norms":true + } + } + }, + { + "string_fields": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "string", + "index": "analyzed", + "omit_norms": true, + "fields": { + "raw": { + "type": "string", + "index": "not_analyzed", + "ignore_above": 256 + } + } + } + } + } + ], + "properties": { + "message": { + "type": "string", + "index": "analyzed" + }, + "exceptions": { + "type": "nested", + "properties": { + "Depth": { + "type": "integer" + }, + "RemoteStackIndex": { + "type": "integer" + }, + "HResult": { + "type": "integer" + }, + "StackTraceString": { + "type": "string", + "index": "analyzed" + }, + "RemoteStackTraceString": { + "type": "string", + "index": "analyzed" + }, + "ExceptionMessage": { + "type": "object", + "properties": { + "MemberType": { + "type": "integer" + } + } + } + } + } + } + } + } +} \ No newline at end of file From c5c91bb5b6cd9c98527aba8b36fa244e7e0bd6e4 Mon Sep 17 00:00:00 2001 From: Michiel van Oudheusden Date: Mon, 16 Dec 2019 11:12:37 +0100 Subject: [PATCH 3/6] Added more tests --- .../Serilog.Sinks.Elasticsearch.Tests.csproj | 2 + .../Templating/SetElasticsearchSinkOptions.cs | 26 ++++++ .../Templating/SetTwoShardsInTemplateTests.cs | 66 +++++++++++++++ .../Templating/template_2shards.json | 83 +++++++++++++++++++ 4 files changed, 177 insertions(+) create mode 100644 test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetElasticsearchSinkOptions.cs create mode 100644 test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetTwoShardsInTemplateTests.cs create mode 100644 test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_2shards.json diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj b/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj index 8874a3f9..a45dc5e8 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj @@ -25,6 +25,7 @@ + @@ -34,6 +35,7 @@ + diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetElasticsearchSinkOptions.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetElasticsearchSinkOptions.cs new file mode 100644 index 00000000..ba4e24da --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetElasticsearchSinkOptions.cs @@ -0,0 +1,26 @@ +using System; +using FluentAssertions; +using Xunit; + +namespace Serilog.Sinks.Elasticsearch.Tests.Templating +{ + public class SetElasticsearchSinkOptions : ElasticsearchSinkTestsBase + { + + [Fact] + public void WhenCreatingOptions_NumberOfShardsInjected_NumberOfShardsAreSet() + { + var options = new ElasticsearchSinkOptions(new Uri("http://localhost:9100")) + { + AutoRegisterTemplate = true, + + NumberOfShards = 2, + NumberOfReplicas = 0 + }; + + options.NumberOfReplicas.Should().Be(0); + options.NumberOfShards.Should().Be(2); + + } + } +} \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetTwoShardsInTemplateTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetTwoShardsInTemplateTests.cs new file mode 100644 index 00000000..4a06c4d6 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetTwoShardsInTemplateTests.cs @@ -0,0 +1,66 @@ +using System; +using System.Reflection; +using FluentAssertions; +using Newtonsoft.Json.Linq; +using Xunit; + +namespace Serilog.Sinks.Elasticsearch.Tests.Templating +{ + public class SetTwoShardsInTemplateTests : ElasticsearchSinkTestsBase + { + private readonly Tuple _templatePut; + + public SetTwoShardsInTemplateTests() + { + _options.AutoRegisterTemplate = true; + _options.NumberOfShards = 2; + _options.NumberOfReplicas= 0; + + var loggerConfig = new LoggerConfiguration() + .MinimumLevel.Debug() + .Enrich.WithMachineName() + .WriteTo.ColoredConsole() + .WriteTo.Elasticsearch(_options); + + var logger = loggerConfig.CreateLogger(); + using (logger as IDisposable) + { + logger.Error("Test exception. Should not contain an embedded exception object."); + } + + this._seenHttpPosts.Should().NotBeNullOrEmpty().And.HaveCount(1); + this._seenHttpPuts.Should().NotBeNullOrEmpty().And.HaveCount(1); + _templatePut = this._seenHttpPuts[0]; + } + + [Fact] + public void ShouldRegisterTheCorrectTemplateOnRegistration() + { + var method = typeof(SendsTemplateTests).GetMethod(nameof(ShouldRegisterTheCorrectTemplateOnRegistration)); + JsonEquals(_templatePut.Item2, method, "template"); + } + + [Fact] + public void TemplatePutToCorrectUrl() + { + var uri = _templatePut.Item1; + uri.AbsolutePath.Should().Be("/_template/serilog-events-template"); + } + + protected void JsonEquals(string json, MethodBase method, string fileName = null) + { +#if DOTNETCORE + var assembly = typeof(SendsTemplateTests).GetTypeInfo().Assembly; +#else + var assembly = Assembly.GetExecutingAssembly(); +#endif + var expected = TestDataHelper.ReadEmbeddedResource(assembly, "template_2shards.json"); + + var nJson = JObject.Parse(json); + var nOtherJson = JObject.Parse(expected); + var equals = JToken.DeepEquals(nJson, nOtherJson); + if (equals) return; + expected.Should().BeEquivalentTo(json); + } + } +} \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_2shards.json b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_2shards.json new file mode 100644 index 00000000..82a097f1 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_2shards.json @@ -0,0 +1,83 @@ +{ + "template": "logstash-*", + "settings": { + "index.refresh_interval": "5s", + "number_of_shards": "2", + "number_of_replicas": "0" + }, + "mappings": { + "_default_": { + "_all": { + "enabled": true, + "omit_norms" : true + }, + "dynamic_templates": [ + { + "numerics_in_fields": { + "path_match":"fields\\.[\\d+]$", + "match_pattern":"regex", + "mapping": { + "type":"string", + "index":"analyzed", + "omit_norms":true + } + } + }, + { + "string_fields": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "string", + "index": "analyzed", + "omit_norms": true, + "fields": { + "raw": { + "type": "string", + "index": "not_analyzed", + "ignore_above": 256 + } + } + } + } + } + ], + "properties": { + "message": { + "type": "string", + "index": "analyzed" + }, + "exceptions": { + "type": "nested", + "properties": { + "Depth": { + "type": "integer" + }, + "RemoteStackIndex": { + "type": "integer" + }, + "HResult": { + "type": "integer" + }, + "StackTraceString": { + "type": "string", + "index": "analyzed" + }, + "RemoteStackTraceString": { + "type": "string", + "index": "analyzed" + }, + "ExceptionMessage": { + "type": "object", + "properties": { + "MemberType": { + "type": "integer" + } + } + } + } + } + } + } + } +} \ No newline at end of file From ecf7adc09b029b6f755aa7ee10a3a513b7b4b9d8 Mon Sep 17 00:00:00 2001 From: Michiel van Oudheusden Date: Mon, 16 Dec 2019 13:57:35 +0100 Subject: [PATCH 4/6] Fix force lowercase indexname, closes #306 --- .../ElasticSearch/ElasticsearchSinkOptions.cs | 3 +- .../ElasticSearch/ElasticsearchSinkState.cs | 4 +- .../CustomIndexTypeNameTests.cs | 41 +++++++++++++++++++ 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs index 41119fe9..94885684 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs @@ -92,6 +92,7 @@ public class ElasticsearchSinkOptions /// /// The index name formatter. A string.Format using the DateTimeOffset of the event is run over this string. /// defaults to "logstash-{0:yyyy.MM.dd}" + /// Needs to be lowercased. /// public string IndexFormat { get; set; } @@ -382,7 +383,7 @@ public enum RegisterTemplateRecovery IndexAnyway = 1, ///// - ///// Keep buffering the data until it is written. be aware you might hit a limit here. + ///// Keep buffering the data until it is written. be aware you might hit a limit here. ///// //BufferUntilSuccess = 2, diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs index 7c775632..6d07d821 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs @@ -82,8 +82,8 @@ private ElasticsearchSinkState(ElasticsearchSinkOptions options) _templateName = options.TemplateName; _templateMatchString = IndexFormatRegex.Replace(options.IndexFormat, @"$1*$2"); - _indexDecider = options.IndexDecider ?? ((@event, offset) => string.Format(options.IndexFormat, offset)); - _bufferedIndexDecider = options.BufferIndexDecider ?? ((@event, offset) => string.Format(options.IndexFormat, offset)); + _indexDecider = options.IndexDecider ?? ((@event, offset) => string.Format(options.IndexFormat, offset).ToLowerInvariant()); + _bufferedIndexDecider = options.BufferIndexDecider ?? ((@event, offset) => string.Format(options.IndexFormat, offset).ToLowerInvariant()); _options = options; diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/CustomIndexTypeNameTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/CustomIndexTypeNameTests.cs index 12d4952c..d3f44b53 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/CustomIndexTypeNameTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/CustomIndexTypeNameTests.cs @@ -52,6 +52,47 @@ public void CustomIndex_And_TypeName_EndsUpInTheOutput() } + [Fact] + public void UpperCasedIndex_And_TypeName_EndsUpInTheOutput() + { + //DO NOTE that you cant send objects as scalar values through Logger.*("{Scalar}", {}); + var timestamp = new DateTimeOffset(2013, 05, 28, 22, 10, 20, 666, TimeSpan.FromHours(10)); + const string messageTemplate = "{Song}++ @{Complex}"; + var template = new MessageTemplateParser().Parse(messageTemplate); + _options.TypeName = "custom-event-type"; + _options.IndexFormat = "Event-Index-{0:yyyy.MM.dd}"; + using (var sink = new ElasticsearchSink(_options)) + { + var properties = new List { new LogEventProperty("Song", new ScalarValue("New Macabre")) }; + var e = new LogEvent(timestamp, LogEventLevel.Information, null, template, properties); + //one off + sink.Emit(e); + + sink.Emit(e); + var exception = new ArgumentException("parameter"); + properties = new List + { + new LogEventProperty("Song", new ScalarValue("Old Macabre")), + new LogEventProperty("Complex", new ScalarValue(new { A = 1, B = 2})) + }; + e = new LogEvent(timestamp.AddYears(-2), LogEventLevel.Fatal, exception, template, properties); + sink.Emit(e); + } + + var bulkJsonPieces = this.AssertSeenHttpPosts(_seenHttpPosts, 4); + + bulkJsonPieces.Should().HaveCount(4); + bulkJsonPieces[0].Should().Contain(@"""_index"":""event-index-2013.05.28"); + bulkJsonPieces[0].Should().Contain(@"""_type"":""custom-event-type"); + bulkJsonPieces[1].Should().Contain("New Macabre"); + bulkJsonPieces[2].Should().Contain(@"""_index"":""event-index-2011.05.28"); + bulkJsonPieces[2].Should().Contain(@"""_type"":""custom-event-type"); + bulkJsonPieces[3].Should().Contain("Old Macabre"); + + bulkJsonPieces[3].Should().Contain("Complex\":{"); + + } + } } From 7ad2b1b5d314c3e9b4f4587715c6fc135e1e3b62 Mon Sep 17 00:00:00 2001 From: Michiel van Oudheusden Date: Mon, 16 Dec 2019 22:12:54 +0100 Subject: [PATCH 5/6] Update sample, switch to ES7.5 --- .env | 4 +- docker-compose.yml | 5 +- .../Program.cs | 96 ++++--------------- .../Serilog.Sinks.Elasticsearch.Sample.csproj | 10 +- .../ElasticSearch/ElasticsearchSinkState.cs | 10 +- 5 files changed, 33 insertions(+), 92 deletions(-) diff --git a/.env b/.env index 664970c7..f7d926f7 100644 --- a/.env +++ b/.env @@ -1,3 +1,3 @@ -TAG=6.1.3 -ELASTIC_VERSION=6.1.3 +TAG=7.5.0 +ELASTIC_VERSION=7.5.0 ELASTIC_PASSWORD=changeme \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 288b6e59..64d6d078 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,10 +16,11 @@ services: mem_limit: 1g environment: - cluster.name=docker-cluster + - discovery.type=single-node - bootstrap.memory_lock=true - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - - discovery.type=zen - - discovery.zen.ping.unicast.hosts=elasticsearch + - node.master=true + - node.data=true - http.cors.enabled=true - http.cors.allow-origin=* - http.cors.allow-headers=Content-Type, Access-Control-Allow-Headers, Authorization, X-Requested-With diff --git a/sample/Serilog.Sinks.Elasticsearch.Sample/Program.cs b/sample/Serilog.Sinks.Elasticsearch.Sample/Program.cs index 2681091e..56b314a8 100644 --- a/sample/Serilog.Sinks.Elasticsearch.Sample/Program.cs +++ b/sample/Serilog.Sinks.Elasticsearch.Sample/Program.cs @@ -23,6 +23,7 @@ class Program .AddJsonFile("appsettings.json", true, true) .AddEnvironmentVariables() .Build(); + static void Main(string[] args) { @@ -31,28 +32,24 @@ static void Main(string[] args) Log.Logger = new LoggerConfiguration() .MinimumLevel.Debug() .WriteTo.Console(theme: SystemConsoleTheme.Literate) - //not persistant - .WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri(Configuration.GetConnectionString("elasticsearch"))) // for the docker-compose implementation - { - AutoRegisterTemplate = true, - //BufferBaseFilename = "./buffer", - RegisterTemplateFailure = RegisterTemplateRecovery.IndexAnyway, - FailureCallback = e => Console.WriteLine("Unable to submit event " + e.MessageTemplate), - EmitEventFailure = EmitEventFailureHandling.WriteToSelfLog | - EmitEventFailureHandling.WriteToFailureSink | - EmitEventFailureHandling.RaiseCallback, - FailureSink = new FileSink("./fail-{Date}.txt", new JsonFormatter(), null, null) - }) - .CreateLogger(); - - //SetupLoggerWithSimplePersistantStorage(); - //LoggingLevelSwitch levelSwitch = SetupLoggerWithPersistantStorage(); + .WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri(Configuration.GetConnectionString("elasticsearch"))) // for the docker-compose implementation + { + AutoRegisterTemplate = true, + DetectElasticsearchVersion = true, + NumberOfReplicas = 0, + NumberOfShards = 2, + //BufferBaseFilename = "./buffer", + RegisterTemplateFailure = RegisterTemplateRecovery.FailSink, + FailureCallback = e => Console.WriteLine("Unable to submit event " + e.MessageTemplate), + EmitEventFailure = EmitEventFailureHandling.WriteToSelfLog | + EmitEventFailureHandling.WriteToFailureSink | + EmitEventFailureHandling.RaiseCallback, + FailureSink = new FileSink("./fail-{Date}.txt", new JsonFormatter(), null, null) + }) + .CreateLogger(); - //Log.Debug("To high loglevel default is Information this will not be logged"); - //levelSwitch.MinimumLevel = LogEventLevel.Debug; Log.Information("Hello, world!"); - //Log.Information("To big log row bigger than SingleEventSizePostingLimit ! {a}", new string('*', 5000)); - + int a = 10, b = 0; try { @@ -75,63 +72,6 @@ static void Main(string[] args) } } - private static LoggingLevelSwitch SetupLoggerWithPersistantStorage() - { - //persistant storage with all settings available for this mode - //please note that all limit settings here is set verry low for test, default values should usually work best! - var levelSwitch = new LoggingLevelSwitch(); - Log.Logger = new LoggerConfiguration() - .MinimumLevel.Debug() - .WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri(Configuration.GetConnectionString("elasticsearch"))) - { - AutoRegisterTemplate = true, - BufferBaseFilename = "./buffer/logserilog", - IndexFormat = "log-serilog-{0:yyyy.MM}", - RegisterTemplateFailure = RegisterTemplateRecovery.FailSink, - BufferCleanPayload = (failingEvent, statuscode, exception) => - { - dynamic e = JObject.Parse(failingEvent); - return JsonConvert.SerializeObject(new Dictionary() - { - { "@timestamp",e["@timestamp"]}, - { "level",e.level}, - { "message","Error: "+e.message}, - { "messageTemplate",e.messageTemplate}, - { "failingStatusCode", statuscode}, - { "failingException", exception} - }); - }, - OverwriteTemplate = true, - NumberOfShards = 1, - NumberOfReplicas = 1, - GetTemplateContent = null, - AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv6, - PipelineName = null, - TypeName = "logevent", - BufferIndexDecider = (logEvent, offset) => "log-serilog-" + (new Random().Next(0, 2)), - BatchPostingLimit = 50, - BufferLogShippingInterval = TimeSpan.FromSeconds(5), - SingleEventSizePostingLimit = 1000, - LevelSwitch = levelSwitch, - BufferRetainedInvalidPayloadsLimitBytes = 2000, - BufferFileSizeLimitBytes = 2000, - BufferFileCountLimit = 2 - }) - .CreateLogger(); - return levelSwitch; - } - - private static void SetupLoggerWithSimplePersistantStorage() - { - //presistant - Log.Logger = new LoggerConfiguration() - .MinimumLevel.Debug() - .WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri(Configuration.GetConnectionString("elasticsearch"))) - { - BufferBaseFilename = "./buffer/logserilogsimple", - IndexFormat = "log-serilog-simple-{0:yyyy.MM}" - }) - .CreateLogger(); - } + } } diff --git a/sample/Serilog.Sinks.Elasticsearch.Sample/Serilog.Sinks.Elasticsearch.Sample.csproj b/sample/Serilog.Sinks.Elasticsearch.Sample/Serilog.Sinks.Elasticsearch.Sample.csproj index 56d53189..20f60c18 100644 --- a/sample/Serilog.Sinks.Elasticsearch.Sample/Serilog.Sinks.Elasticsearch.Sample.csproj +++ b/sample/Serilog.Sinks.Elasticsearch.Sample/Serilog.Sinks.Elasticsearch.Sample.csproj @@ -2,14 +2,14 @@ Exe - netcoreapp2.0 + netcoreapp2.1 - - - - + + + + diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs index 6d07d821..fe2af093 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs @@ -203,13 +203,13 @@ public void RegisterTemplateIfNeeded() private PostData GetTemplatePostData() { - //PostData no longer exposes an implict cast from object. Previously it supported that and would inspect the object Type to - //determine if it it was a litteral string to write directly or if it was an object that it needed to serialse. Now the onus is - //on us to tell it what type we are passing otherwise if the user specified the template as a json string it would be serialised again. + //PostData no longer exposes an implicit cast from object. Previously it supported that and would inspect the object Type to + //determine if it it was a literal string to write directly or if it was an object that it needed to serialize. Now the onus is + //on us to tell it what type we are passing otherwise if the user specified the template as a json string it would be serialized again. var template = GetTemplateData(); - if (template is string) + if (template is string s) { - return PostData.String((string)template); + return PostData.String(s); } else { From 89c0cc66aa77a5ef1892c3e5df55e2498d56bc27 Mon Sep 17 00:00:00 2001 From: Michiel van Oudheusden Date: Mon, 16 Dec 2019 22:48:30 +0100 Subject: [PATCH 6/6] Updated sample app --- CHANGES.md | 4 ++++ sample/Serilog.Sinks.Elasticsearch.Sample/Program.cs | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 034debfb..245c286a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,9 @@ ## Changelog +8.1 + * Updated sample to use .NET 2.1 and ES 7.5 + * Change default TypeName to '_doc' #298 + 8.0 * Adds Elasticsearch 7.0 support #256 * Adds DetectElasticsearchVersion to the sink that will detect the running cluster version. Something we now use to support sending Esv6 templates to Elasticsearch 7.x and Esv7 templates to Elasticsearch 6.x which should simplify upgrades. diff --git a/sample/Serilog.Sinks.Elasticsearch.Sample/Program.cs b/sample/Serilog.Sinks.Elasticsearch.Sample/Program.cs index 56b314a8..565a0be5 100644 --- a/sample/Serilog.Sinks.Elasticsearch.Sample/Program.cs +++ b/sample/Serilog.Sinks.Elasticsearch.Sample/Program.cs @@ -35,8 +35,10 @@ static void Main(string[] args) .WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri(Configuration.GetConnectionString("elasticsearch"))) // for the docker-compose implementation { AutoRegisterTemplate = true, + OverwriteTemplate = true, DetectElasticsearchVersion = true, - NumberOfReplicas = 0, + AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7, + NumberOfReplicas = 1, NumberOfShards = 2, //BufferBaseFilename = "./buffer", RegisterTemplateFailure = RegisterTemplateRecovery.FailSink,