diff --git a/CHANGES.md b/CHANGES.md index 922b692c..f23db77b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,9 @@ == Changelog +6.3 + * Render message by default (#160). + * Expose interface-typed options via appsettings (#162) + 6.2 * Extra overload added to support more settings via AppSettings reader. (#150) diff --git a/README.md b/README.md index f33351df..80979c25 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ Register the sink in code or using the appSettings reader (from v2.0.42+) as sho var loggerConfig = new LoggerConfiguration() .WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri("http://localhost:9200") ){ AutoRegisterTemplate = true, + AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv6 }); ``` @@ -49,10 +50,27 @@ This example shows the options that are currently available when using the appSe + + + + + + + + + + + + + + + + + ``` -With the appSettings configuration the `nodeUris` property is required. Multiple nodes can be specified using `,` or `;` to seperate them. All other properties are optional. +With the appSettings configuration the `nodeUris` property is required. Multiple nodes can be specified using `,` or `;` to seperate them. All other properties are optional. Also required is the '' setting to include this sink. All other properties are optional. If you do not explicitly specify an indexFormat-setting, a generic index such as 'logstash-[current_date]' will be used automatically. And start writing your events using Serilog. @@ -123,7 +141,24 @@ In your `appsettings.json` file, under the `Serilog` node, : "bufferBaseFilename": "C:/Temp/LogDigipolis/docker-elk-serilog-web-buffer", "bufferFileSizeLimitBytes": 5242880, "bufferLogShippingInterval": 5000, - "connectionGlobalHeaders" :"Authorization=Bearer SOME-TOKEN;OtherHeader=OTHER-HEADER-VALUE" + "connectionGlobalHeaders" :"Authorization=Bearer SOME-TOKEN;OtherHeader=OTHER-HEADER-VALUE", + "connectionTimeout": 5, + "emitEventFailure": "WriteToSelfLog", + "queueSizeLimit": "100000", + "autoRegisterTemplate": true, + "autoRegisterTemplateVersion": "ESv2", + "overwriteTemplate": false, + "registerTemplateFailure": "IndexAnyway", + "deadLetterIndexName": "deadletter-{0:yyyy.MM}", + "numberOfShards": 20, + "numberOfReplicas": 10, + "formatProvider": "My.Namespace.MyFormatProvider, My.Assembly.Name", + "connection": "My.Namespace.MyConnection, My.Assembly.Name", + "serializer": "My.Namespace.MySerializer, My.Assembly.Name", + "connectionPool": "My.Namespace.MyConnectionPool, My.Assembly.Name", + "customFormatter": "My.Namespace.MyCustomFormatter, My.Assembly.Name", + "customDurableFormatter": "My.Namespace.MyCustomDurableFormatter, My.Assembly.Name", + "failureSink": "My.Namespace.MyFailureSink, My.Assembly.Name" } }] } @@ -165,7 +200,9 @@ Since version 5.5 you can use the RegisterTemplateFailure option. Set it to one ### Breaking changes for version 6 -Starting from version 6, the sink has been upgraded to work with Elasticsearch 6.0 and has support for the new templates used by ES 6. +Starting from version 6, the sink has been upgraded to work with Elasticsearch 6.0 and has support for the new templates used by ES 6. + +If you use the `AutoRegisterTemplate` option, you need to set the `AutoRegisterTemplateVersion` option to `ESv6` in order to generate default templates that are compatible with the breaking changes in ES 6. ### Breaking changes for version 4 diff --git a/src/Serilog.Sinks.Elasticsearch/LoggerConfigurationElasticSearchExtensions.cs b/src/Serilog.Sinks.Elasticsearch/LoggerConfigurationElasticSearchExtensions.cs index 3a546f47..19dbe7bc 100644 --- a/src/Serilog.Sinks.Elasticsearch/LoggerConfigurationElasticSearchExtensions.cs +++ b/src/Serilog.Sinks.Elasticsearch/LoggerConfigurationElasticSearchExtensions.cs @@ -22,6 +22,8 @@ using Serilog.Sinks.Elasticsearch; using System.Collections.Specialized; using System.ComponentModel; +using Elasticsearch.Net; +using Serilog.Formatting; namespace Serilog { @@ -237,5 +239,162 @@ public static LoggerConfiguration Elasticsearch( return Elasticsearch(loggerSinkConfiguration, options); } + + /// + /// Overload to allow basic configuration through AppSettings. + /// + /// Options for the sink. + /// A comma or semi column separated list of URIs for Elasticsearch nodes. + /// + /// + /// + /// + /// + /// + /// The minimum log event level required in order to write an event to the sink. Ignored when is specified. + /// A switch allowing the pass-through minimum level to be changed at runtime. + /// + /// + /// + /// A comma or semi column separated list of key value pairs of headers to be added to each elastic http request + /// The connection timeout (in seconds) when sending bulk operations to elasticsearch (defaults to 5). + /// Specifies how failing emits should be handled. + /// The maximum number of events that will be held in-memory while waiting to ship them to Elasticsearch. Beyond this limit, events will be dropped. The default is 100,000. Has no effect on durable log shipping. + /// Name the Pipeline where log events are sent to sink. Please note that the Pipeline should be existing before the usage starts. + /// When set to true the sink will register an index template for the logs in elasticsearch. + /// When using the AutoRegisterTemplate feature, this allows to set the Elasticsearch version. Depending on the version, a template will be selected. Defaults to pre 5.0. + /// When using the AutoRegisterTemplate feature, this allows you to overwrite the template in Elasticsearch if it already exists. Defaults to false + /// Specifies the option on how to handle failures when writing the template to Elasticsearch. This is only applicable when using the AutoRegisterTemplate option. + /// Optionally set this value to the name of the index that should be used when the template cannot be written to ES. + /// The default number of shards. + /// The default number of replicas. + /// Supplies culture-specific formatting information, or null. + /// Allows you to override the connection used to communicate with elasticsearch. + /// When passing a serializer unknown object will be serialized to object instead of relying on their ToString representation + /// The connectionpool describing the cluster to write event to + /// Customizes the formatter used when converting log events into ElasticSearch documents. Please note that the formatter output must be valid JSON :) + /// Customizes the formatter used when converting log events into the durable sink. Please note that the formatter output must be valid JSON :) + /// Sink to use when Elasticsearch is unable to accept the events. This is optionally and depends on the EmitEventFailure setting. + /// LoggerConfiguration object + /// is . + public static LoggerConfiguration Elasticsearch( + this LoggerSinkConfiguration loggerSinkConfiguration, + string nodeUris, + string indexFormat = null, + string templateName = null, + string typeName = "logevent", + int batchPostingLimit = 50, + int period = 2, + bool inlineFields = false, + LogEventLevel restrictedToMinimumLevel = LevelAlias.Minimum, + string bufferBaseFilename = null, + long? bufferFileSizeLimitBytes = null, + long bufferLogShippingInterval = 5000, + string connectionGlobalHeaders = null, + LoggingLevelSwitch levelSwitch = null, + int connectionTimeout = 5, + EmitEventFailureHandling emitEventFailure = EmitEventFailureHandling.WriteToSelfLog, + int queueSizeLimit = 100000, + string pipelineName = null, + bool autoRegisterTemplate = false, + AutoRegisterTemplateVersion autoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv2, + bool overwriteTemplate = false, + RegisterTemplateRecovery registerTemplateFailure = RegisterTemplateRecovery.IndexAnyway, + string deadLetterIndexName = null, + int? numberOfShards = null, + int? numberOfReplicas = null, + IFormatProvider formatProvider = null, + IConnection connection = null, + IElasticsearchSerializer serializer = null, + IConnectionPool connectionPool = null, + ITextFormatter customFormatter = null, + ITextFormatter customDurableFormatter = null, + ILogEventSink failureSink = null) + { + if (string.IsNullOrEmpty(nodeUris)) + throw new ArgumentNullException(nameof(nodeUris), "No Elasticsearch node(s) specified."); + + IEnumerable nodes = nodeUris + .Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries) + .Select(uriString => new Uri(uriString)); + + var options = connectionPool == null ? new ElasticsearchSinkOptions(nodes) : new ElasticsearchSinkOptions(connectionPool); + + if (!string.IsNullOrWhiteSpace(indexFormat)) + { + options.IndexFormat = indexFormat; + } + + if (!string.IsNullOrWhiteSpace(templateName)) + { + options.AutoRegisterTemplate = true; + options.TemplateName = templateName; + } + + if (!string.IsNullOrWhiteSpace(typeName)) + { + options.TypeName = typeName; + } + + options.BatchPostingLimit = batchPostingLimit; + options.Period = TimeSpan.FromSeconds(period); + options.InlineFields = inlineFields; + options.MinimumLogEventLevel = restrictedToMinimumLevel; + options.LevelSwitch = levelSwitch; + + if (!string.IsNullOrWhiteSpace(bufferBaseFilename)) + { + Path.GetFullPath(bufferBaseFilename); // validate path + options.BufferBaseFilename = bufferBaseFilename; + } + + if (bufferFileSizeLimitBytes.HasValue) + { + options.BufferFileSizeLimitBytes = bufferFileSizeLimitBytes.Value; + } + + options.BufferLogShippingInterval = TimeSpan.FromMilliseconds(bufferLogShippingInterval); + + if (!string.IsNullOrWhiteSpace(connectionGlobalHeaders)) + { + NameValueCollection headers = new NameValueCollection(); + connectionGlobalHeaders + .Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries) + .ToList() + .ForEach(headerValueStr => + { + var headerValue = headerValueStr.Split(new[] { '=' }, 2, StringSplitOptions.RemoveEmptyEntries); + headers.Add(headerValue[0], headerValue[1]); + }); + + options.ModifyConnectionSettings = (c) => c.GlobalHeaders(headers); + } + + options.ConnectionTimeout = TimeSpan.FromSeconds(connectionTimeout); + options.EmitEventFailure = emitEventFailure; + options.QueueSizeLimit = queueSizeLimit; + options.PipelineName = pipelineName; + + options.AutoRegisterTemplate = autoRegisterTemplate; + options.AutoRegisterTemplateVersion = autoRegisterTemplateVersion; + options.RegisterTemplateFailure = registerTemplateFailure; + options.OverwriteTemplate = overwriteTemplate; + options.NumberOfShards = numberOfShards; + options.NumberOfReplicas = numberOfReplicas; + + if (!string.IsNullOrWhiteSpace(deadLetterIndexName)) + { + options.DeadLetterIndexName = deadLetterIndexName; + } + + options.FormatProvider = formatProvider; + options.FailureSink = failureSink; + options.Connection = connection; + options.CustomFormatter = customFormatter; + options.CustomDurableFormatter = customDurableFormatter; + options.Serializer = serializer; + + return Elasticsearch(loggerSinkConfiguration, options); + } } } diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/DefaultJsonFormatter.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/DefaultJsonFormatter.cs index bcc4ef61..11301de6 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/DefaultJsonFormatter.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/DefaultJsonFormatter.cs @@ -35,6 +35,7 @@ public abstract class DefaultJsonFormatter : ITextFormatter readonly bool _omitEnclosingObject; readonly string _closingDelimiter; readonly bool _renderMessage; + readonly bool _renderMessageTemplate; readonly IFormatProvider _formatProvider; readonly IDictionary> _literalWriters; @@ -50,15 +51,19 @@ public abstract class DefaultJsonFormatter : ITextFormatter /// If true, the message will be rendered and written to the output as a /// property named RenderedMessage. /// Supplies culture-specific formatting information, or null. + /// If true, the message template will be rendered and written to the output as a + /// property named RenderedMessageTemplate. protected DefaultJsonFormatter( bool omitEnclosingObject = false, string closingDelimiter = null, - bool renderMessage = false, - IFormatProvider formatProvider = null) + bool renderMessage = true, + IFormatProvider formatProvider = null, + bool renderMessageTemplate = true) { _omitEnclosingObject = omitEnclosingObject; _closingDelimiter = closingDelimiter ?? Environment.NewLine; _renderMessage = renderMessage; + _renderMessageTemplate = renderMessageTemplate; _formatProvider = formatProvider; _literalWriters = new Dictionary> @@ -102,7 +107,12 @@ public void Format(LogEvent logEvent, TextWriter output) var delim = ""; WriteTimestamp(logEvent.Timestamp, ref delim, output); WriteLevel(logEvent.Level, ref delim, output); - WriteMessageTemplate(logEvent.MessageTemplate.Text, ref delim, output); + + if(_renderMessageTemplate) + { + WriteMessageTemplate(logEvent.MessageTemplate.Text, ref delim, output); + } + if (_renderMessage) { var message = logEvent.RenderMessage(_formatProvider); diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchJsonFormatter.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchJsonFormatter.cs index d8cb8cd6..bd16bff9 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchJsonFormatter.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchJsonFormatter.cs @@ -33,6 +33,27 @@ public class ElasticsearchJsonFormatter : DefaultJsonFormatter readonly IElasticsearchSerializer _serializer; readonly bool _inlineFields; + /// + /// Render message property name + /// + public const string RenderedMessagePropertyName = "message"; + /// + /// Message template property name + /// + public const string MessageTemplatePropertyName = "messageTemplate"; + /// + /// Exception property name + /// + public const string ExceptionPropertyName = "Exception"; + /// + /// Level property name + /// + public const string LevelPropertyName = "level"; + /// + /// Timestamp property name + /// + public const string TimestampPropertyName = "@timestamp"; + /// /// Construct a . /// @@ -47,13 +68,17 @@ public class ElasticsearchJsonFormatter : DefaultJsonFormatter /// Supplies culture-specific formatting information, or null. /// Inject a serializer to force objects to be serialized over being ToString() /// When set to true values will be written at the root of the json document - public ElasticsearchJsonFormatter(bool omitEnclosingObject = false, + /// If true, the message template will be rendered and written to the output as a + /// property named RenderedMessageTemplate. + public ElasticsearchJsonFormatter( + bool omitEnclosingObject = false, string closingDelimiter = null, - bool renderMessage = false, + bool renderMessage = true, IFormatProvider formatProvider = null, IElasticsearchSerializer serializer = null, - bool inlineFields = false) - : base(omitEnclosingObject, closingDelimiter, renderMessage, formatProvider) + bool inlineFields = false, + bool renderMessageTemplate = true) + : base(omitEnclosingObject, closingDelimiter, renderMessage, formatProvider, renderMessageTemplate) { _serializer = serializer; _inlineFields = inlineFields; @@ -240,13 +265,12 @@ private void WriteStructuredExceptionMethod(string exceptionMethodString, ref st delim = ","; } - /// /// (Optionally) writes out the rendered message /// protected override void WriteRenderedMessage(string message, ref string delim, TextWriter output) { - WriteJsonProperty("message", message, ref delim, output); + WriteJsonProperty(RenderedMessagePropertyName, message, ref delim, output); } /// @@ -254,7 +278,7 @@ protected override void WriteRenderedMessage(string message, ref string delim, T /// protected override void WriteMessageTemplate(string template, ref string delim, TextWriter output) { - WriteJsonProperty("messageTemplate", template, ref delim, output); + WriteJsonProperty(MessageTemplatePropertyName, template, ref delim, output); } /// @@ -263,7 +287,7 @@ protected override void WriteMessageTemplate(string template, ref string delim, protected override void WriteLevel(LogEventLevel level, ref string delim, TextWriter output) { var stringLevel = Enum.GetName(typeof(LogEventLevel), level); - WriteJsonProperty("level", stringLevel, ref delim, output); + WriteJsonProperty(LevelPropertyName, stringLevel, ref delim, output); } /// @@ -271,7 +295,7 @@ protected override void WriteLevel(LogEventLevel level, ref string delim, TextWr /// protected override void WriteTimestamp(DateTimeOffset timestamp, ref string delim, TextWriter output) { - WriteJsonProperty("@timestamp", timestamp, ref delim, output); + WriteJsonProperty(TimestampPropertyName, timestamp, ref delim, output); } /// diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs index 28bdd52c..d454e07d 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs @@ -223,7 +223,7 @@ public int QueueSizeLimit /// /// Configures the elasticsearch sink defaults /// - protected ElasticsearchSinkOptions() + public ElasticsearchSinkOptions() { this.IndexFormat = "logstash-{0:yyyy.MM.dd}"; this.DeadLetterIndexName = "deadletter-{0:yyyy.MM.dd}"; diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs index 49dadddc..2b277186 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs @@ -77,27 +77,34 @@ private ElasticsearchSinkState(ElasticsearchSinkOptions options) _client = new ElasticLowLevelClient(configuration); - _formatter = options.CustomFormatter ?? new ElasticsearchJsonFormatter( + _formatter = options.CustomFormatter ?? CreateDefaultFormatter(options); + + _durableFormatter = options.CustomDurableFormatter ?? CreateDefaultDurableFormatter(options); + + _registerTemplateOnStartup = options.AutoRegisterTemplate; + TemplateRegistrationSuccess = !_registerTemplateOnStartup; + } + + public static ITextFormatter CreateDefaultFormatter(ElasticsearchSinkOptions options) + { + return new ElasticsearchJsonFormatter( formatProvider: options.FormatProvider, - renderMessage: true, closingDelimiter: string.Empty, serializer: options.Serializer, inlineFields: options.InlineFields ); + } - _durableFormatter = options.CustomDurableFormatter ?? new ElasticsearchJsonFormatter( + public static ITextFormatter CreateDefaultDurableFormatter(ElasticsearchSinkOptions options) + { + return new ElasticsearchJsonFormatter( formatProvider: options.FormatProvider, - renderMessage: true, closingDelimiter: Environment.NewLine, serializer: options.Serializer, inlineFields: options.InlineFields ); - - _registerTemplateOnStartup = options.AutoRegisterTemplate; - TemplateRegistrationSuccess = !_registerTemplateOnStartup; } - public string Serialize(object o) { return _client.Serializer.SerializeToString(o, SerializationFormatting.None); diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchJsonFormatterTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchJsonFormatterTests.cs new file mode 100644 index 00000000..7b225884 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchJsonFormatterTests.cs @@ -0,0 +1,137 @@ +using Serilog.Events; +using Serilog.Parsing; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using Xunit; + +namespace Serilog.Sinks.Elasticsearch.Tests +{ + public class ElasticsearchJsonFormatterTests + { + #region Helpers + static LogEvent CreateLogEvent() => + new LogEvent + ( + DateTimeOffset.Now, + LogEventLevel.Debug, + exception: null, + messageTemplate: new MessageTemplate(Enumerable.Empty()), + properties: Enumerable.Empty() + ); + + static void CheckProperties(Func logEventProvider, ElasticsearchJsonFormatter formatter, Action assert) + { + string result = null; + + var logEvent = logEventProvider(); + + using (var stringWriter = new StringWriter()) + { + formatter.Format(logEvent, stringWriter); + + result = stringWriter.ToString(); + } + + assert(result); + } + + static void CheckProperties(ElasticsearchJsonFormatter formatter, Action assert) => + CheckProperties(CreateLogEvent, formatter, assert); + + static void ContainsProperty(string propertyToCheck, string result) => + Assert.Contains + ( + propertyToCheck, + result, + StringComparison.CurrentCultureIgnoreCase + ); + static void DoesNotContainsProperty(string propertyToCheck, string result) => + Assert.DoesNotContain + ( + propertyToCheck, + result, + StringComparison.CurrentCultureIgnoreCase + ); + + static string FormatProperty(string property) => $"\"{property}\":"; + #endregion + + [Theory] + [InlineData(ElasticsearchJsonFormatter.RenderedMessagePropertyName)] + [InlineData(ElasticsearchJsonFormatter.MessageTemplatePropertyName)] + [InlineData(ElasticsearchJsonFormatter.TimestampPropertyName)] + [InlineData(ElasticsearchJsonFormatter.LevelPropertyName)] + public void DefaultJsonFormater_Should_Render_default_properties(string propertyToCheck) + { + CheckProperties( + new ElasticsearchJsonFormatter(), + result => ContainsProperty(FormatProperty(propertyToCheck), result)); + } + + [Fact] + public void When_disabling_renderMessage_Should_not_render_message_but_others() + { + CheckProperties( + new ElasticsearchJsonFormatter(renderMessage: false), + result => + { + DoesNotContainsProperty(FormatProperty(ElasticsearchJsonFormatter.RenderedMessagePropertyName), result); + ContainsProperty(FormatProperty(ElasticsearchJsonFormatter.MessageTemplatePropertyName), result); + ContainsProperty(FormatProperty(ElasticsearchJsonFormatter.TimestampPropertyName), result); + ContainsProperty(FormatProperty(ElasticsearchJsonFormatter.LevelPropertyName), result); + }); + } + + [Fact] + public void When_disabling_renderMessageTemplate_Should_not_render_message_template_but_others() + { + CheckProperties( + new ElasticsearchJsonFormatter(renderMessageTemplate: false), + result => + { + DoesNotContainsProperty(FormatProperty(ElasticsearchJsonFormatter.MessageTemplatePropertyName), result); + ContainsProperty(FormatProperty(ElasticsearchJsonFormatter.RenderedMessagePropertyName), result); + ContainsProperty(FormatProperty(ElasticsearchJsonFormatter.TimestampPropertyName), result); + ContainsProperty(FormatProperty(ElasticsearchJsonFormatter.LevelPropertyName), result); + }); + } + + [Fact] + public void DefaultJsonFormater_Should_enclose_object() + { + CheckProperties( + new ElasticsearchJsonFormatter(), + result => + { + Assert.StartsWith("{", result); + Assert.EndsWith($"}}{Environment.NewLine}", result); + }); + } + + [Fact] + public void When_omitEnclosingObject_should_not_enclose_object() + { + CheckProperties( + new ElasticsearchJsonFormatter(omitEnclosingObject: true), + result => + { + Assert.StartsWith("\"", result); + Assert.EndsWith("\"", result); + }); + } + + [Fact] + public void When_provide_closing_delimiter_should_use_it() + { + CheckProperties( + new ElasticsearchJsonFormatter(closingDelimiter: "closingDelimiter"), + result => + { + Assert.EndsWith("closingDelimiter", result); + }); + } + } +}