Merge dev to master (#168)

* Explain use of `AutoRegisterTemplateVersion` in the README (#157)

It took me a while to figure out that the reason the sink failed to start with a somewhat cryptic "Failed to execute" error was that, while I had updated all libraries to ES6-compatibile versions, I had not set the  `AutoRegisterTemplateVersion` option and so the template sent was not ES6-compatible (it used "string").

This PR explains this snag in the readme.

For a future PR, would it make sense to deprecate the manual option and automatically detect the ES version when attempting to register a template? It can be fetched by a GET request at the root URL, which returns:

```    {
      "name" : "8hV7EYy",
      "cluster_name" : "docker-cluster",
      "cluster_uuid" : "bC1bTjVuTiyWoaprIIZIIw",
      "version" : {
        "number" : "6.2.1",
        "build_hash" : "7299dc3",
        "build_date" : "2018-02-07T19:34:26.990113Z",
        "build_snapshot" : false,
        "lucene_version" : "7.2.1",
        "minimum_wire_compatibility_version" : "5.6.0",
        "minimum_index_compatibility_version" : "5.0.0"
      },
      "tagline" : "You Know, for Search"
    }
```

* Added some explanation

See #165

* Render message by default #158, Could disable MessageTemplate #159 + Tests (#160)

* Updated readme

* Expose interface-typed options via appsettings (#162)

* Expose interface-typed settings via appsettings

* Amend readme to show changes for json appsettings as well

* Updated changelog

CHANGES.md

@@ -1,5 +1,9 @@
 == Changelog
 
+6.3
+ * Render message by default (#160). 
+ * Expose interface-typed options via appsettings (#162)
+
 6.2
  * Extra overload added to support more settings via AppSettings reader. (#150)
 

README.md

@@ -28,6 +28,7 @@ Register the sink in code or using the appSettings reader (from v2.0.42+) as sho
 var loggerConfig = new LoggerConfiguration()
     .WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri("http://localhost:9200") ){
              AutoRegisterTemplate = true,
+             AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv6
      });
 ```
 
@@ -49,10 +50,27 @@ This example shows the options that are currently available when using the appSe
     <add key="serilog:write-to:Elasticsearch.bufferFileSizeLimitBytes" value="5242880"/>
     <add key="serilog:write-to:Elasticsearch.bufferLogShippingInterval" value="5000"/>	
     <add key="serilog:write-to:Elasticsearch.connectionGlobalHeaders" value="Authorization=Bearer SOME-TOKEN;OtherHeader=OTHER-HEADER-VALUE" />
+    <add key="serilog:write-to:Elasticsearch.connectionTimeout" value="5" />
+    <add key="serilog:write-to:Elasticsearch.emitEventFailure" value="WriteToSelfLog" />
+    <add key="serilog:write-to:Elasticsearch.queueSizeLimit" value="100000" />
+    <add key="serilog:write-to:Elasticsearch.autoRegisterTemplate" value="true" />
+    <add key="serilog:write-to:Elasticsearch.autoRegisterTemplateVersion" value="ESv2" />
+    <add key="serilog:write-to:Elasticsearch.overwriteTemplate" value="false" />
+    <add key="serilog:write-to:Elasticsearch.registerTemplateFailure" value="IndexAnyway" />
+    <add key="serilog:write-to:Elasticsearch.deadLetterIndexName" value="deadletter-{0:yyyy.MM}" />
+    <add key="serilog:write-to:Elasticsearch.numberOfShards" value="20" />
+    <add key="serilog:write-to:Elasticsearch.numberOfReplicas" value="10" />
+    <add key="serilog:write-to:Elasticsearch.formatProvider" value="My.Namespace.MyFormatProvider, My.Assembly.Name" />
+    <add key="serilog:write-to:Elasticsearch.connection" value="My.Namespace.MyConnection, My.Assembly.Name" />
+    <add key="serilog:write-to:Elasticsearch.serializer" value="My.Namespace.MySerializer, My.Assembly.Name" />
+    <add key="serilog:write-to:Elasticsearch.connectionPool" value="My.Namespace.MyConnectionPool, My.Assembly.Name" />
+    <add key="serilog:write-to:Elasticsearch.customFormatter" value="My.Namespace.MyCustomFormatter, My.Assembly.Name" />
+    <add key="serilog:write-to:Elasticsearch.customDurableFormatter" value="My.Namespace.MyCustomDurableFormatter, My.Assembly.Name" />
+    <add key="serilog:write-to:Elasticsearch.failureSink" value="My.Namespace.MyFailureSink, My.Assembly.Name" />
   </appSettings>
 ```
 
-With the appSettings configuration the `nodeUris` property is required. Multiple nodes can be specified using `,` or `;` to seperate them. All other properties are optional.
+With the appSettings configuration the `nodeUris` property is required. Multiple nodes can be specified using `,` or `;` to seperate them. All other properties are optional. Also required is the '<add key="serilog:using" value="Serilog.Sinks.Elasticsearch"/>' setting to include this sink. All other properties are optional. If you do not explicitly specify an indexFormat-setting, a generic index such as 'logstash-[current_date]' will be used automatically.
 
 And start writing your events using Serilog.
 
@@ -123,7 +141,24 @@ In your `appsettings.json` file, under the `Serilog` node, :
           "bufferBaseFilename":  "C:/Temp/LogDigipolis/docker-elk-serilog-web-buffer",
           "bufferFileSizeLimitBytes": 5242880,
           "bufferLogShippingInterval": 5000,
-          "connectionGlobalHeaders" :"Authorization=Bearer SOME-TOKEN;OtherHeader=OTHER-HEADER-VALUE"
+          "connectionGlobalHeaders" :"Authorization=Bearer SOME-TOKEN;OtherHeader=OTHER-HEADER-VALUE",
+          "connectionTimeout": 5,
+          "emitEventFailure": "WriteToSelfLog",
+          "queueSizeLimit": "100000",
+          "autoRegisterTemplate": true,
+          "autoRegisterTemplateVersion": "ESv2",
+          "overwriteTemplate": false,
+          "registerTemplateFailure": "IndexAnyway",
+          "deadLetterIndexName": "deadletter-{0:yyyy.MM}",
+          "numberOfShards": 20,
+          "numberOfReplicas": 10,
+          "formatProvider": "My.Namespace.MyFormatProvider, My.Assembly.Name",
+          "connection": "My.Namespace.MyConnection, My.Assembly.Name",
+          "serializer": "My.Namespace.MySerializer, My.Assembly.Name",
+          "connectionPool": "My.Namespace.MyConnectionPool, My.Assembly.Name",
+          "customFormatter": "My.Namespace.MyCustomFormatter, My.Assembly.Name",
+          "customDurableFormatter": "My.Namespace.MyCustomDurableFormatter, My.Assembly.Name",
+          "failureSink": "My.Namespace.MyFailureSink, My.Assembly.Name"
         }       
     }]
   }
@@ -165,7 +200,9 @@ Since version 5.5 you can use the RegisterTemplateFailure option. Set it to one
 
 ### Breaking changes for version 6
 
-Starting from version 6, the sink has been upgraded to work with Elasticsearch 6.0 and has support for the new templates used by ES 6.
+Starting from version 6, the sink has been upgraded to work with Elasticsearch 6.0 and has support for the new templates used by ES 6. 
+
+If you use the `AutoRegisterTemplate` option, you need to set the `AutoRegisterTemplateVersion` option to `ESv6` in order to generate default templates that are compatible with the breaking changes in ES 6.
 
 ### Breaking changes for version 4
 

src/Serilog.Sinks.Elasticsearch/LoggerConfigurationElasticSearchExtensions.cs

@@ -22,6 +22,8 @@
 using Serilog.Sinks.Elasticsearch;
 using System.Collections.Specialized;
 using System.ComponentModel;
+using Elasticsearch.Net;
+using Serilog.Formatting;
 
 namespace Serilog
 {
@@ -237,5 +239,162 @@ public static LoggerConfiguration Elasticsearch(
 
             return Elasticsearch(loggerSinkConfiguration, options);
         }
+
+        /// <summary>
+        /// Overload to allow basic configuration through AppSettings.
+        /// </summary>
+        /// <param name="loggerSinkConfiguration">Options for the sink.</param>
+        /// <param name="nodeUris">A comma or semi column separated list of URIs for Elasticsearch nodes.</param>
+        /// <param name="indexFormat"><see cref="ElasticsearchSinkOptions.IndexFormat"/></param>
+        /// <param name="templateName"><see cref="ElasticsearchSinkOptions.TemplateName"/></param>
+        /// <param name="typeName"><see cref="ElasticsearchSinkOptions.TypeName"/></param>
+        /// <param name="batchPostingLimit"><see cref="ElasticsearchSinkOptions.BatchPostingLimit"/></param>
+        /// <param name="period"><see cref="ElasticsearchSinkOptions.Period"/></param>
+        /// <param name="inlineFields"><see cref="ElasticsearchSinkOptions.InlineFields"/></param>
+        /// <param name="restrictedToMinimumLevel">The minimum log event level required in order to write an event to the sink. Ignored when <paramref name="levelSwitch"/> is specified.</param>
+        /// <param name="levelSwitch">A switch allowing the pass-through minimum level to be changed at runtime.</param>
+        /// <param name="bufferBaseFilename"><see cref="ElasticsearchSinkOptions.BufferBaseFilename"/></param>
+        /// <param name="bufferFileSizeLimitBytes"><see cref="ElasticsearchSinkOptions.BufferFileSizeLimitBytes"/></param>
+        /// <param name="bufferLogShippingInterval"><see cref="ElasticsearchSinkOptions.BufferLogShippingInterval"/></param>
+        /// <param name="connectionGlobalHeaders">A comma or semi column separated list of key value pairs of headers to be added to each elastic http request</param>   
+        /// <param name="connectionTimeout"><see cref="ElasticsearchSinkOptions.ConnectionTimeout"/>The connection timeout (in seconds) when sending bulk operations to elasticsearch (defaults to 5).</param>   
+        /// <param name="emitEventFailure"><see cref="ElasticsearchSinkOptions.EmitEventFailure"/>Specifies how failing emits should be handled.</param>  
+        /// <param name="queueSizeLimit"><see cref="ElasticsearchSinkOptions.QueueSizeLimit"/>The maximum number of events that will be held in-memory while waiting to ship them to Elasticsearch. Beyond this limit, events will be dropped. The default is 100,000. Has no effect on durable log shipping.</param>   
+        /// <param name="pipelineName"><see cref="ElasticsearchSinkOptions.PipelineName"/>Name the Pipeline where log events are sent to sink. Please note that the Pipeline should be existing before the usage starts.</param>   
+        /// <param name="autoRegisterTemplate"><see cref="ElasticsearchSinkOptions.AutoRegisterTemplate"/>When set to true the sink will register an index template for the logs in elasticsearch.</param>   
+        /// <param name="autoRegisterTemplateVersion"><see cref="ElasticsearchSinkOptions.AutoRegisterTemplateVersion"/>When using the AutoRegisterTemplate feature, this allows to set the Elasticsearch version. Depending on the version, a template will be selected. Defaults to pre 5.0.</param>  
+        /// <param name="overwriteTemplate"><see cref="ElasticsearchSinkOptions.OverwriteTemplate"/>When using the AutoRegisterTemplate feature, this allows you to overwrite the template in Elasticsearch if it already exists. Defaults to false</param>   
+        /// <param name="registerTemplateFailure"><see cref="ElasticsearchSinkOptions.RegisterTemplateFailure"/>Specifies the option on how to handle failures when writing the template to Elasticsearch. This is only applicable when using the AutoRegisterTemplate option.</param>  
+        /// <param name="deadLetterIndexName"><see cref="ElasticsearchSinkOptions.DeadLetterIndexName"/>Optionally set this value to the name of the index that should be used when the template cannot be written to ES.</param>  
+        /// <param name="numberOfShards"><see cref="ElasticsearchSinkOptions.NumberOfShards"/>The default number of shards.</param>   
+        /// <param name="numberOfReplicas"><see cref="ElasticsearchSinkOptions.NumberOfReplicas"/>The default number of replicas.</param>
+        /// <param name="formatProvider">Supplies culture-specific formatting information, or null.</param>
+        /// <param name="connection">Allows you to override the connection used to communicate with elasticsearch.</param>
+        /// <param name="serializer">When passing a serializer unknown object will be serialized to object instead of relying on their ToString representation</param>
+        /// <param name="connectionPool">The connectionpool describing the cluster to write event to</param>
+        /// <param name="customFormatter">Customizes the formatter used when converting log events into ElasticSearch documents. Please note that the formatter output must be valid JSON :)</param>
+        /// <param name="customDurableFormatter">Customizes the formatter used when converting log events into the durable sink. Please note that the formatter output must be valid JSON :)</param>
+        /// <param name="failureSink">Sink to use when Elasticsearch is unable to accept the events. This is optionally and depends on the EmitEventFailure setting.</param>   
+        /// <returns>LoggerConfiguration object</returns>
+        /// <exception cref="ArgumentNullException"><paramref name="nodeUris"/> is <see langword="null" />.</exception>
+        public static LoggerConfiguration Elasticsearch(
+            this LoggerSinkConfiguration loggerSinkConfiguration,
+            string nodeUris,
+            string indexFormat = null,
+            string templateName = null,
+            string typeName = "logevent",
+            int batchPostingLimit = 50,
+            int period = 2,
+            bool inlineFields = false,
+            LogEventLevel restrictedToMinimumLevel = LevelAlias.Minimum,
+            string bufferBaseFilename = null,
+            long? bufferFileSizeLimitBytes = null,
+            long bufferLogShippingInterval = 5000,
+            string connectionGlobalHeaders = null,
+            LoggingLevelSwitch levelSwitch = null,
+            int connectionTimeout = 5,
+            EmitEventFailureHandling emitEventFailure = EmitEventFailureHandling.WriteToSelfLog,
+            int queueSizeLimit = 100000,
+            string pipelineName = null,
+            bool autoRegisterTemplate = false,
+            AutoRegisterTemplateVersion autoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv2,
+            bool overwriteTemplate = false,
+            RegisterTemplateRecovery registerTemplateFailure = RegisterTemplateRecovery.IndexAnyway,
+            string deadLetterIndexName = null,
+            int? numberOfShards = null,
+            int? numberOfReplicas = null,
+            IFormatProvider formatProvider = null,
+            IConnection connection = null,
+            IElasticsearchSerializer serializer = null,
+            IConnectionPool connectionPool = null,
+            ITextFormatter customFormatter = null,
+            ITextFormatter customDurableFormatter = null,
+            ILogEventSink failureSink = null)
+        {
+            if (string.IsNullOrEmpty(nodeUris))
+                throw new ArgumentNullException(nameof(nodeUris), "No Elasticsearch node(s) specified.");
+
+            IEnumerable<Uri> nodes = nodeUris
+                .Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries)
+                .Select(uriString => new Uri(uriString));
+
+            var options = connectionPool == null ? new ElasticsearchSinkOptions(nodes) : new ElasticsearchSinkOptions(connectionPool);
+
+            if (!string.IsNullOrWhiteSpace(indexFormat))
+            {
+                options.IndexFormat = indexFormat;
+            }
+
+            if (!string.IsNullOrWhiteSpace(templateName))
+            {
+                options.AutoRegisterTemplate = true;
+                options.TemplateName = templateName;
+            }
+
+            if (!string.IsNullOrWhiteSpace(typeName))
+            {
+                options.TypeName = typeName;
+            }
+
+            options.BatchPostingLimit = batchPostingLimit;
+            options.Period = TimeSpan.FromSeconds(period);
+            options.InlineFields = inlineFields;
+            options.MinimumLogEventLevel = restrictedToMinimumLevel;
+            options.LevelSwitch = levelSwitch;
+
+            if (!string.IsNullOrWhiteSpace(bufferBaseFilename))
+            {
+                Path.GetFullPath(bufferBaseFilename);       // validate path
+                options.BufferBaseFilename = bufferBaseFilename;
+            }
+
+            if (bufferFileSizeLimitBytes.HasValue)
+            {
+                options.BufferFileSizeLimitBytes = bufferFileSizeLimitBytes.Value;
+            }
+
+            options.BufferLogShippingInterval = TimeSpan.FromMilliseconds(bufferLogShippingInterval);
+
+            if (!string.IsNullOrWhiteSpace(connectionGlobalHeaders))
+            {
+                NameValueCollection headers = new NameValueCollection();
+                connectionGlobalHeaders
+                    .Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries)
+                    .ToList()
+                    .ForEach(headerValueStr =>
+                    {
+                        var headerValue = headerValueStr.Split(new[] { '=' }, 2, StringSplitOptions.RemoveEmptyEntries);
+                        headers.Add(headerValue[0], headerValue[1]);
+                    });
+
+                options.ModifyConnectionSettings = (c) => c.GlobalHeaders(headers);
+            }
+
+            options.ConnectionTimeout = TimeSpan.FromSeconds(connectionTimeout);
+            options.EmitEventFailure = emitEventFailure;
+            options.QueueSizeLimit = queueSizeLimit;
+            options.PipelineName = pipelineName;
+
+            options.AutoRegisterTemplate = autoRegisterTemplate;
+            options.AutoRegisterTemplateVersion = autoRegisterTemplateVersion;
+            options.RegisterTemplateFailure = registerTemplateFailure;
+            options.OverwriteTemplate = overwriteTemplate;
+            options.NumberOfShards = numberOfShards;
+            options.NumberOfReplicas = numberOfReplicas;
+
+            if (!string.IsNullOrWhiteSpace(deadLetterIndexName))
+            {
+                options.DeadLetterIndexName = deadLetterIndexName;
+            }
+
+            options.FormatProvider = formatProvider;
+            options.FailureSink = failureSink;
+            options.Connection = connection;
+            options.CustomFormatter = customFormatter;
+            options.CustomDurableFormatter = customDurableFormatter;
+            options.Serializer = serializer;
+
+            return Elasticsearch(loggerSinkConfiguration, options);
+        }
     }
 }

src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/DefaultJsonFormatter.cs

@@ -35,6 +35,7 @@ public abstract class DefaultJsonFormatter : ITextFormatter
         readonly bool _omitEnclosingObject;
         readonly string _closingDelimiter;
         readonly bool _renderMessage;
+        readonly bool _renderMessageTemplate;
         readonly IFormatProvider _formatProvider;
         readonly IDictionary<Type, Action<object, bool, TextWriter>> _literalWriters;
 
@@ -50,15 +51,19 @@ public abstract class DefaultJsonFormatter : ITextFormatter
         /// <param name="renderMessage">If true, the message will be rendered and written to the output as a
         /// property named RenderedMessage.</param>
         /// <param name="formatProvider">Supplies culture-specific formatting information, or null.</param>
+        /// <param name="renderMessageTemplate">If true, the message template will be rendered and written to the output as a
+        /// property named RenderedMessageTemplate.</param>
         protected DefaultJsonFormatter(
             bool omitEnclosingObject = false,
             string closingDelimiter = null,
-            bool renderMessage = false,
-            IFormatProvider formatProvider = null)
+            bool renderMessage = true,
+            IFormatProvider formatProvider = null,
+            bool renderMessageTemplate = true)
         {
             _omitEnclosingObject = omitEnclosingObject;
             _closingDelimiter = closingDelimiter ?? Environment.NewLine;
             _renderMessage = renderMessage;
+            _renderMessageTemplate = renderMessageTemplate;
             _formatProvider = formatProvider;
 
             _literalWriters = new Dictionary<Type, Action<object, bool, TextWriter>>
@@ -102,7 +107,12 @@ public void Format(LogEvent logEvent, TextWriter output)
             var delim = "";
             WriteTimestamp(logEvent.Timestamp, ref delim, output);
             WriteLevel(logEvent.Level, ref delim, output);
-            WriteMessageTemplate(logEvent.MessageTemplate.Text, ref delim, output);
+
+            if(_renderMessageTemplate)
+            {
+                WriteMessageTemplate(logEvent.MessageTemplate.Text, ref delim, output);
+            }
+
             if (_renderMessage)
             {
                 var message = logEvent.RenderMessage(_formatProvider);