Skip to content
This repository has been archived by the owner on Jun 1, 2024. It is now read-only.

Commit

Permalink
Get rid of 'deprecated' warnings in Elastic 5.6.2 (#142)
Browse files Browse the repository at this point in the history 
* Get rid of 'deprecated' warnings in Elastic 5.6.2

Our Elastic 5.6.2 logs are flooded with below errors:
The [string] field is deprecated, please use [text] or [keyword]
[omit_norms] is deprecated, please use [norms] instead with the opposite boolean value

Updated the schema following https://www.elastic.co/blog/strings-are-dead-long-live-strings

* Update ElasticsearchSinkState.cs

* Update template.json

* Update ElasticsearchSinkState.cs

* Update template.json

* add template provider

* fix unit tests
  • Loading branch information
@VanHuychemG @mivano
VanHuychemG authored and mivano committed Jan 18, 2018
1 parent cb9f3e2 commit 4d2425b
Show file tree
Hide file tree
Showing 10 changed files with 757 additions and 124 deletions.
336 changes: 336 additions & 0 deletions src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,336 @@
using System;
using System.Collections.Generic;

namespace Serilog.Sinks.Elasticsearch
{
/// <summary>
///
/// </summary>
public enum AutoRegisterTemplateVersion
{
/// <summary>
/// Elasticsearch version &lt;= 2.4
/// </summary>
ESv2 = 0,
/// <summary>
/// Elasticsearch version &lt;= version 5.6
/// </summary>
ESv5 = 1,
/// <summary>
/// Elasticsearch version &gt;= version 6.0
/// </summary>
ESv6 = 2
}

/// <summary>
///
/// </summary>
public class ElasticSearchTemplateProvider
{
/// <summary>
///
/// </summary>
/// <param name="settings"></param>
/// <param name="templateMatchString"></param>
/// <param name="version"></param>
/// <returns></returns>
public static object GetTemplate(
Dictionary<string, string> settings,
string templateMatchString,
AutoRegisterTemplateVersion version = AutoRegisterTemplateVersion.ESv2)
{
switch (version)
{
case AutoRegisterTemplateVersion.ESv5:
return GetTemplateESv5(settings, templateMatchString);
case AutoRegisterTemplateVersion.ESv6:
return GetTemplateESv6(settings, templateMatchString);
case AutoRegisterTemplateVersion.ESv2:
return GetTemplateESv2(settings, templateMatchString);
default:
throw new ArgumentOutOfRangeException(nameof(version), version, null);
}
}

private static object GetTemplateESv6(
Dictionary<string, string> settings,
string templateMatchString)
{
return new
{
template = templateMatchString,
settings = settings,
mappings = new
{
_default_ = new
{
dynamic_templates = new List<Object>
{
//when you use serilog as an adaptor for third party frameworks
//where you have no control over the log message they typically
//contain {0} ad infinitum, we force numeric property names to
//contain strings by default.
{
new
{
numerics_in_fields = new
{
path_match = @"fields\.[\d+]$",
match_pattern = "regex",
mapping = new
{
type = "text",
index = true,
norms = false
}
}
}
},
{
new
{
string_fields = new
{
match = "*",
match_mapping_type = "string",
mapping = new
{
type = "text",
index = true,
norms = false,
fields = new
{
raw = new
{
type = "keyword",
index = true,
ignore_above = 256
}
}
}
}
}
}
},
properties = new Dictionary<string, object>
{
{"message", new {type = "text", index = "analyzed"}},
{
"exceptions", new
{
type = "nested",
properties = new Dictionary<string, object>
{
{"Depth", new {type = "integer"}},
{"RemoteStackIndex", new {type = "integer"}},
{"HResult", new {type = "integer"}},
{"StackTraceString", new {type = "text", index = "analyzed"}},
{"RemoteStackTraceString", new {type = "text", index = "analyzed"}},
{
"ExceptionMessage", new
{
type = "object",
properties = new Dictionary<string, object>
{
{"MemberType", new {type = "integer"}},
}
}
}
}
}
}
}
}
}
};
}

private static object GetTemplateESv5(
Dictionary<string, string> settings,
string templateMatchString)
{
return new
{
template = templateMatchString,
settings = settings,
mappings = new
{
_default_ = new
{
_all = new { enabled = true, norms = false },
dynamic_templates = new List<Object>
{
//when you use serilog as an adaptor for third party frameworks
//where you have no control over the log message they typically
//contain {0} ad infinitum, we force numeric property names to
//contain strings by default.
{
new
{
numerics_in_fields = new
{
path_match = @"fields\.[\d+]$",
match_pattern = "regex",
mapping = new
{
type = "text",
index = true,
norms = false
}
}
}
},
{
new
{
string_fields = new
{
match = "*",
match_mapping_type = "string",
mapping = new
{
type = "text",
index = true,
norms = false,
fields = new
{
raw = new
{
type = "keyword",
index = true,
ignore_above = 256
}
}
}
}
}
}
},
properties = new Dictionary<string, object>
{
{"message", new {type = "text", index = "analyzed"}},
{
"exceptions", new
{
type = "nested",
properties = new Dictionary<string, object>
{
{"Depth", new {type = "integer"}},
{"RemoteStackIndex", new {type = "integer"}},
{"HResult", new {type = "integer"}},
{"StackTraceString", new {type = "text", index = "analyzed"}},
{"RemoteStackTraceString", new {type = "text", index = "analyzed"}},
{
"ExceptionMessage", new
{
type = "object",
properties = new Dictionary<string, object>
{
{"MemberType", new {type = "integer"}},
}
}
}
}
}
}
}
}
}
};
}

private static object GetTemplateESv2(
Dictionary<string, string> settings,
string templateMatchString)
{
return new
{
template = templateMatchString,
settings = settings,
mappings = new
{
_default_ = new
{
_all = new { enabled = true, omit_norms = true },
dynamic_templates = new List<Object>
{
//when you use serilog as an adaptor for third party frameworks
//where you have no control over the log message they typically
//contain {0} ad infinitum, we force numeric property names to
//contain strings by default.
{
new
{
numerics_in_fields = new
{
path_match = @"fields\.[\d+]$",
match_pattern = "regex",
mapping = new
{
type = "string",
index = "analyzed",
omit_norms = true
}
}
}
},
{
new
{
string_fields = new
{
match = "*",
match_mapping_type = "string",
mapping = new
{
type = "string",
index = "analyzed",
omit_norms = true,
fields = new
{
raw = new
{
type = "string",
index = "not_analyzed",
ignore_above = 256
}
}
}
}
}
}
},
properties = new Dictionary<string, object>
{
{"message", new {type = "string", index = "analyzed"}},
{
"exceptions", new
{
type = "nested",
properties = new Dictionary<string, object>
{
{"Depth", new {type = "integer"}},
{"RemoteStackIndex", new {type = "integer"}},
{"HResult", new {type = "integer"}},
{"StackTraceString", new {type = "string", index = "analyzed"}},
{"RemoteStackTraceString", new {type = "string", index = "analyzed"}},
{
"ExceptionMessage", new
{
type = "object",
properties = new Dictionary<string, object>
{
{"MemberType", new {type = "integer"}},
}
}
}
}
}
}
}
}
}
};
}
}
}
16 changes: 11 additions & 5 deletions src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
// Copyright 2014 Serilog Contributors
//
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//
// http://www.apache.org/licenses/LICENSE-2.0
//
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Expand Down Expand Up @@ -35,6 +35,12 @@ public class ElasticsearchSinkOptions
/// </summary>
public bool AutoRegisterTemplate { get; set; }

/// <summary>
/// When using the <see cref="AutoRegisterTemplate"/> feature, this allows to set the Elasticsearch version. Depending on the
/// version, a template will be selected. Defaults to pre 5.0.
/// </summary>
public AutoRegisterTemplateVersion AutoRegisterTemplateVersion { get; set; }

/// <summary>
/// Specifies the option on how to handle failures when writing the template to Elasticsearch. This is only applicable when using the AutoRegisterTemplate option.
/// </summary>
Expand Down Expand Up @@ -173,7 +179,7 @@ public class ElasticsearchSinkOptions
public ITextFormatter CustomDurableFormatter { get; set; }

/// <summary>
/// Specifies how failing emits should be handled.
/// Specifies how failing emits should be handled.
/// </summary>
public EmitEventFailureHandling EmitEventFailure { get; set; }

Expand Down Expand Up @@ -292,7 +298,7 @@ public enum RegisterTemplateRecovery
IndexAnyway = 1,

///// <summary>
///// Keep buffering the data until it is written. be aware you might hit a limit here.
///// Keep buffering the data until it is written. be aware you might hit a limit here.
///// </summary>
//BufferUntilSuccess = 2,

Expand Down
Loading

0 comments on commit 4d2425b

Please sign in to comment.