Skip to content
This repository has been archived by the owner on Jun 1, 2024. It is now read-only.

Commit

Permalink
Serilog.Skinks.Elasticsearch with V7 compatibility
Browse files Browse the repository at this point in the history
  • Loading branch information
@kanadaj
kanadaj committed May 29, 2019
1 parent 729b506 commit 549f791
Show file tree
Hide file tree
Showing 8 changed files with 212 additions and 10 deletions.
2 changes: 1 addition & 1 deletion global.json
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"sdk": {
"version": "2.1.4"
"version": "2.1.500"
}
}
10 changes: 9 additions & 1 deletion src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
</ItemGroup>

<ItemGroup>
<PackageReference Include="Elasticsearch.Net" Version="6.0.0" />
<PackageReference Include="Serilog" Version="2.8.0" />
<PackageReference Include="Serilog.Formatting.Compact" Version="1.0.0" />
<PackageReference Include="Serilog.Sinks.File" Version="4.0.0" />
Expand All @@ -43,7 +42,16 @@
<ItemGroup Condition=" '$(TargetFramework)' == 'net45' ">
<Reference Include="System" />
<Reference Include="Microsoft.CSharp" />
<PackageReference Include="Elasticsearch.Net" Version="6.0.0" />
</ItemGroup>
<ItemGroup Condition=" '$(TargetFramework)' == 'netstandard1.3' ">
<PackageReference Include="Elasticsearch.Net" Version="6.0.0" />
</ItemGroup>

<ItemGroup Condition=" '$(TargetFramework)' == 'netstandard2.0' ">
<PackageReference Include="Elasticsearch.Net" Version="7.0.0-alpha2" />
<PackageReference Include="Microsoft.CSharp" Version="4.5.0" />
</ItemGroup>

<PropertyGroup Condition=" '$(TargetFramework)' == 'netstandard1.3' ">
<NoWarn>1591;1701;1702</NoWarn>
Expand Down
6 changes: 3 additions & 3 deletions src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ protected override void EmitBatch(IEnumerable<LogEvent> events)
var items = result.Body["items"];
foreach (var item in items)
{
if (item.index != null && item.index.error != null)
if (item["index"] != null && item["index"]["error"] != null)
{
var e = events.ElementAt(indexer);
if (_state.Options.EmitEventFailure.HasFlag(EmitEventFailureHandling.WriteToSelfLog))
Expand All @@ -80,8 +80,8 @@ protected override void EmitBatch(IEnumerable<LogEvent> events)
SelfLog.WriteLine(
"Failed to store event with template '{0}' into Elasticsearch. Elasticsearch reports for index {1} the following: {2}",
e.MessageTemplate,
item.index._index,
item.index.error);
item["index"]["_index"],
_state.Serialize(item["index"]["error"]));
}

if (_state.Options.EmitEventFailure.HasFlag(EmitEventFailureHandling.WriteToFailureSink) &&
Expand Down
98 changes: 97 additions & 1 deletion src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,11 @@ public enum AutoRegisterTemplateVersion
/// <summary>
/// Elasticsearch version &gt;= version 6.0
/// </summary>
ESv6 = 2
ESv6 = 2,
/// <summary>
/// Elasticsearch version &gt;= version 7.0
/// </summary>
ESv7 = 3
}

/// <summary>
Expand All @@ -45,13 +49,105 @@ public static object GetTemplate(
return GetTemplateESv5(settings, templateMatchString);
case AutoRegisterTemplateVersion.ESv6:
return GetTemplateESv6(settings, templateMatchString);
case AutoRegisterTemplateVersion.ESv7:
return GetTemplateESv7(settings, templateMatchString);
case AutoRegisterTemplateVersion.ESv2:
return GetTemplateESv2(settings, templateMatchString);
default:
throw new ArgumentOutOfRangeException(nameof(version), version, null);
}
}

private static object GetTemplateESv7(
Dictionary<string, string> settings,
string templateMatchString)
{
return new
{
index_patterns = new[] { templateMatchString },
settings = settings,
mappings = new
{
dynamic_templates = new List<Object>
{
//when you use serilog as an adaptor for third party frameworks
//where you have no control over the log message they typically
//contain {0} ad infinitum, we force numeric property names to
//contain strings by default.
{
new
{
numerics_in_fields = new
{
path_match = @"fields\.[\d+]$",
match_pattern = "regex",
mapping = new
{
type = "text",
index = true,
norms = false
}
}
}
},
{
new
{
string_fields = new
{
match = "*",
match_mapping_type = "string",
mapping = new
{
type = "text",
index = true,
norms = false,
fields = new
{
raw = new
{
type = "keyword",
index = true,
ignore_above = 256
}
}
}
}
}
}
},
properties = new Dictionary<string, object>
{
{"message", new {type = "text", index = "true"}},
{
"exceptions", new
{
type = "nested",
properties = new Dictionary<string, object>
{
{"Depth", new {type = "integer"}},
{"RemoteStackIndex", new {type = "integer"}},
{"HResult", new {type = "integer"}},
{"StackTraceString", new {type = "text", index = "true"}},
{"RemoteStackTraceString", new {type = "text", index = "true"}},
{
"ExceptionMessage", new
{
type = "object",
properties = new Dictionary<string, object>
{
{"MemberType", new {type = "integer"}},
}
}
}
}
}
}
}
}
};
}

private static object GetTemplateESv6(
Dictionary<string, string> settings,
string templateMatchString)
Expand Down
15 changes: 13 additions & 2 deletions src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,18 @@ public static ElasticsearchSinkState Create(ElasticsearchSinkOptions options)
private ElasticsearchSinkState(ElasticsearchSinkOptions options)
{
if (string.IsNullOrWhiteSpace(options.IndexFormat)) throw new ArgumentException("options.IndexFormat");
if (string.IsNullOrWhiteSpace(options.TypeName)) throw new ArgumentException("options.TypeName");
if (string.IsNullOrWhiteSpace(options.TemplateName)) throw new ArgumentException("options.TemplateName");

// Strip type argument if ESv7 since multiple types are not supported anymore
if (options.AutoRegisterTemplateVersion == AutoRegisterTemplateVersion.ESv7)
{
options.TypeName = "_doc";
}
else
{
if (string.IsNullOrWhiteSpace(options.TypeName)) throw new ArgumentException("options.TypeName");
}

_templateName = options.TemplateName;
_templateMatchString = IndexFormatRegex.Replace(options.IndexFormat, @"$1*$2");

Expand All @@ -70,6 +79,7 @@ private ElasticsearchSinkState(ElasticsearchSinkOptions options)

_options = options;


var configuration = new ConnectionConfiguration(options.ConnectionPool, options.Connection, options.Serializer)
.RequestTimeout(options.ConnectionTimeout);

Expand Down Expand Up @@ -112,7 +122,7 @@ public static ITextFormatter CreateDefaultDurableFormatter(ElasticsearchSinkOpti

public string Serialize(object o)
{
return _client.Serializer.SerializeToString(o, SerializationFormatting.None);
return _client.Serializer.SerializeToString(o, formatting: SerializationFormatting.None);
}

public string GetIndexForEvent(LogEvent e, DateTimeOffset offset)
Expand Down Expand Up @@ -149,6 +159,7 @@ public void RegisterTemplateIfNeeded()
}
}

Console.WriteLine(_client.Serializer.SerializeToString(GetTemplateData()));
var result = _client.IndicesPutTemplateForAll<DynamicResponse>(_templateName, GetTempatePostData());

if (!result.Success)
Expand Down
2 changes: 1 addition & 1 deletion src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/SerializerAdapter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public Task SerializeAsync<T>(T data, Stream stream,

public string SerializeToString(object value)
{
return _elasticsearchSerializer.SerializeToString(value, SerializationFormatting.None);
return _elasticsearchSerializer.SerializeToString(value, formatting: SerializationFormatting.None);
}
}
}
14 changes: 13 additions & 1 deletion test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,11 @@
<None Remove="Templating\template_v2.json" />
<None Remove="Templating\template_v5.json" />
<None Remove="Templating\template_v6.json" />
<None Remove="Templating\template_v7.json" />
</ItemGroup>

<ItemGroup>
<EmbeddedResource Include="Templating\template_v7.json" />
<EmbeddedResource Include="Templating\template_v6.json" />
<EmbeddedResource Include="Templating\template_v5.json" />
<EmbeddedResource Include="Templating\template_v2.json" />
Expand All @@ -39,7 +41,6 @@
</ItemGroup>

<ItemGroup>
<PackageReference Include="Elasticsearch.Net" Version="6.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.5.0" />
<PackageReference Include="Microsoft.NETCore.Platforms" Version="2.0.1" />
<PackageReference Include="NEST.JsonNetSerializer" Version="6.0.0" />
Expand All @@ -61,8 +62,19 @@
<Reference Include="System.Threading.Tasks" />
<Reference Include="System" />
<Reference Include="Microsoft.CSharp" />
<PackageReference Include="Elasticsearch.Net" Version="6.0.0" />
</ItemGroup>


<ItemGroup Condition=" '$(TargetFramework)' == 'netstandard1.3' ">
<PackageReference Include="Elasticsearch.Net" Version="6.0.0" />
</ItemGroup>

<ItemGroup Condition=" '$(TargetFramework)' == 'netstandard2.0' ">
<PackageReference Include="Elasticsearch.Net" Version="7.0.0-alpha2" />
<PackageReference Include="Microsoft.CSharp" Version="4.5.0" />
</ItemGroup>

<PropertyGroup Condition=" '$(TargetFramework)' == 'netcoreapp1.1' ">
<DefineConstants>$(DefineConstants);DOTNETCORE;NO_SERIALIZATION</DefineConstants>
</PropertyGroup>
Expand Down
75 changes: 75 additions & 0 deletions test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v7.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
{
"index_patterns": [ "logstash-*" ],
"settings": {
"index.refresh_interval": "5s"
},
"mappings": {
"dynamic_templates": [
{
"numerics_in_fields": {
"path_match": "fields\\.[\\d+]$",
"match_pattern": "regex",
"mapping": {
"type": "text",
"index": true,
"norms": false
}
}
},
{
"string_fields": {
"match": "*",
"match_mapping_type": "string",
"mapping": {
"type": "text",
"index": true,
"norms": false,
"fields": {
"raw": {
"type": "keyword",
"index": true,
"ignore_above": 256
}
}
}
}
}
],
"properties": {
"message": {
"type": "text",
"index": "true"
},
"exceptions": {
"type": "nested",
"properties": {
"Depth": {
"type": "integer"
},
"RemoteStackIndex": {
"type": "integer"
},
"HResult": {
"type": "integer"
},
"StackTraceString": {
"type": "text",
"index": "true"
},
"RemoteStackTraceString": {
"type": "text",
"index": "true"
},
"ExceptionMessage": {
"type": "object",
"properties": {
"MemberType": {
"type": "integer"
}
}
}
}
}
}
}
}

0 comments on commit 549f791

Please sign in to comment.