diff --git a/.editorconfig b/.editorconfig index 28e37c6f..9bdbb34e 100644 --- a/.editorconfig +++ b/.editorconfig @@ -2,4 +2,7 @@ root=true [*] indent_style = space -indent_size = 4 \ No newline at end of file +indent_size = 4 + +[*.csproj] +indent_size = 2 diff --git a/Build.ps1 b/Build.ps1 index 06c478f5..a6c50387 100644 --- a/Build.ps1 +++ b/Build.ps1 @@ -2,6 +2,7 @@ echo "In directory: $PSScriptRoot" $solution = "serilog-sinks-elasticsearch.sln" $test = "test\\Serilog.Sinks.Elasticsearch.Tests\\Serilog.Sinks.Elasticsearch.Tests.csproj" +$testIntegration = "test\\Serilog.Sinks.Elasticsearch.IntegrationTests\\Serilog.Sinks.Elasticsearch.IntegrationTests.csproj" [string[]]$projects = @( ("src\\Serilog.Sinks.Elasticsearch\\Serilog.Sinks.Elasticsearch.csproj"), ("src\\Serilog.Formatting.Elasticsearch\\Serilog.Formatting.Elasticsearch.csproj") @@ -22,6 +23,16 @@ function Invoke-Build() Write-Output "The tests failed" exit 1 } + + Write-Output "Running integration tests" + # Tee-Object forces console redirection on vstest which magically makes Console.WriteLine works again. + # This allows you to see the console out of Elastic.Xunit while its running + & dotnet test $testIntegration -c Release | Tee-Object -Variable integ + if($LASTEXITCODE -ne 0) + { + Write-Output "The integration tests failed" + exit 1 + } Write-Output "Creating packages" foreach ($project in $projects) diff --git a/global.json b/global.json index 341f9fea..99e7444e 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "2.1.4" + "version": "2.1.500" } } \ No newline at end of file diff --git a/nuget.config b/nuget.config index 96116f07..a81fc3ec 100644 --- a/nuget.config +++ b/nuget.config @@ -11,5 +11,7 @@ + + \ No newline at end of file diff --git a/serilog-sinks-elasticsearch.sln b/serilog-sinks-elasticsearch.sln index 1f08366a..551c394d 100644 --- a/serilog-sinks-elasticsearch.sln +++ b/serilog-sinks-elasticsearch.sln @@ -24,6 +24,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serilog.Sinks.Elasticsearch EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serilog.Formatting.Elasticsearch", "src\Serilog.Formatting.Elasticsearch\Serilog.Formatting.Elasticsearch.csproj", "{0E6D34BF-322A-4803-94D1-355F6D5024BE}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serilog.Sinks.Elasticsearch.IntegrationTests", "test\Serilog.Sinks.Elasticsearch.IntegrationTests\Serilog.Sinks.Elasticsearch.IntegrationTests.csproj", "{23BC3821-E028-48B4-8F2C-83BB1B8B5525}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -46,6 +48,10 @@ Global {0E6D34BF-322A-4803-94D1-355F6D5024BE}.Debug|Any CPU.Build.0 = Debug|Any CPU {0E6D34BF-322A-4803-94D1-355F6D5024BE}.Release|Any CPU.ActiveCfg = Release|Any CPU {0E6D34BF-322A-4803-94D1-355F6D5024BE}.Release|Any CPU.Build.0 = Release|Any CPU + {23BC3821-E028-48B4-8F2C-83BB1B8B5525}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {23BC3821-E028-48B4-8F2C-83BB1B8B5525}.Debug|Any CPU.Build.0 = Debug|Any CPU + {23BC3821-E028-48B4-8F2C-83BB1B8B5525}.Release|Any CPU.ActiveCfg = Release|Any CPU + {23BC3821-E028-48B4-8F2C-83BB1B8B5525}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/serilog-sinks-elasticsearch.sln.DotSettings b/serilog-sinks-elasticsearch.sln.DotSettings new file mode 100644 index 00000000..e62f9e23 --- /dev/null +++ b/serilog-sinks-elasticsearch.sln.DotSettings @@ -0,0 +1,8 @@ + + True + True + True + True + True + True + True \ No newline at end of file diff --git a/src/Serilog.Formatting.Elasticsearch/Serilog.Formatting.ElasticSearch.nuspec b/src/Serilog.Formatting.Elasticsearch/Serilog.Formatting.ElasticSearch.nuspec index 3047dcd8..50f0444a 100644 --- a/src/Serilog.Formatting.Elasticsearch/Serilog.Formatting.ElasticSearch.nuspec +++ b/src/Serilog.Formatting.Elasticsearch/Serilog.Formatting.ElasticSearch.nuspec @@ -13,6 +13,7 @@ serilog logging elasticsearch formatters + diff --git a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.Symbols.nuspec b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.Symbols.nuspec index 289c659a..9932b8fb 100644 --- a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.Symbols.nuspec +++ b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.Symbols.nuspec @@ -12,7 +12,7 @@ serilog logging elasticsearch - + diff --git a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.nuspec b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.nuspec index 5c6c8148..c6ba9cc7 100644 --- a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.nuspec +++ b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.nuspec @@ -13,7 +13,7 @@ serilog logging elasticsearch - + diff --git a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj index b1d79191..cbe61dbf 100644 --- a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj +++ b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj @@ -4,7 +4,7 @@ 6.0.0 alpha Michiel van Oudheusden, Martijn Laarman, Mogens Heller Grabe, Serilog Contributors - net45;netstandard1.3;netstandard2.0 + net461;netstandard2.0 true true Serilog.Sinks.Elasticsearch @@ -28,49 +28,31 @@ false - - - - - - - - - - - - - - - - - - - 1591;1701;1702 - $(DefineConstants);DURABLE;THREADING_TIMER - - 1591;1701;1702 $(DefineConstants);DURABLE;THREADING_TIMER - + 1591;1701;1702 $(DefineConstants);DURABLE;THREADING_TIMER;HRESULTS - - 1591;1701;1702 - NU1605 - - - - + + + + + + + + + + + diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/DurableElasticSearchSink.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/DurableElasticSearchSink.cs deleted file mode 100644 index 9ac18737..00000000 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/DurableElasticSearchSink.cs +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2014 Serilog Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Collections.Generic; -using System.Text; -using Elasticsearch.Net; -using Serilog.Core; -using Serilog.Events; - - -namespace Serilog.Sinks.Elasticsearch.Durable -{ - class DurableElasticsearchSink : ILogEventSink, IDisposable - { - // we rely on the date in the filename later! - const string FileNameSuffix = "-.json"; - - readonly Logger _sink; - readonly LogShipper> _shipper; - readonly ElasticsearchSinkState _state; - - public DurableElasticsearchSink(ElasticsearchSinkOptions options) - { - _state = ElasticsearchSinkState.Create(options); - - if (string.IsNullOrWhiteSpace(options.BufferBaseFilename)) - { - throw new ArgumentException("Cannot create the durable ElasticSearch sink without a buffer base file name!"); - } - - - _sink = new LoggerConfiguration() - .MinimumLevel.Verbose() - .WriteTo.File(_state.DurableFormatter, - options.BufferBaseFilename + FileNameSuffix, - rollingInterval: RollingInterval.Day, - fileSizeLimitBytes: options.BufferFileSizeLimitBytes, - rollOnFileSizeLimit: true, - retainedFileCountLimit: options.BufferFileCountLimit, - levelSwitch: _state.Options.LevelSwitch, - encoding: Encoding.UTF8) - .CreateLogger(); - - - var elasticSearchLogClient = new ElasticsearchLogClient( - elasticLowLevelClient: _state.Client, - cleanPayload: _state.Options.BufferCleanPayload); - - var payloadReader = new ElasticsearchPayloadReader( - pipelineName: _state.Options.PipelineName, - typeName:_state.Options.TypeName, - serialize:_state.Serialize, - getIndexForEvent: _state.GetBufferedIndexForEvent - ); - - _shipper = new ElasticsearchLogShipper( - bufferBaseFilename: _state.Options.BufferBaseFilename, - batchPostingLimit: _state.Options.BatchPostingLimit, - period: _state.Options.BufferLogShippingInterval ?? TimeSpan.FromSeconds(5), - eventBodyLimitBytes: _state.Options.SingleEventSizePostingLimit, - levelControlSwitch: _state.Options.LevelSwitch, - logClient: elasticSearchLogClient, - payloadReader: payloadReader, - retainedInvalidPayloadsLimitBytes: _state.Options.BufferRetainedInvalidPayloadsLimitBytes, - bufferSizeLimitBytes: _state.Options.BufferFileSizeLimitBytes, - registerTemplateIfNeeded: _state.RegisterTemplateIfNeeded); - - } - - public void Emit(LogEvent logEvent) - { - // This is a lagging indicator, but the network bandwidth usage benefits - // are worth the ambiguity. - if (_shipper.IsIncluded(logEvent)) - { - _sink.Write(logEvent); - } - } - - public void Dispose() - { - _sink.Dispose(); - _shipper.Dispose(); - } - } -} \ No newline at end of file diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchLogClient.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchLogClient.cs deleted file mode 100644 index 6b2d45bf..00000000 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchLogClient.cs +++ /dev/null @@ -1,125 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.ExceptionServices; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using Elasticsearch.Net; -using Serilog.Debugging; - -namespace Serilog.Sinks.Elasticsearch.Durable -{ - /// - /// - /// - public class ElasticsearchLogClient : ILogClient> - { - private readonly IElasticLowLevelClient _elasticLowLevelClient; - private readonly Func _cleanPayload; - - /// - /// - /// - /// - /// - public ElasticsearchLogClient(IElasticLowLevelClient elasticLowLevelClient, - Func cleanPayload) - { - _elasticLowLevelClient = elasticLowLevelClient; - _cleanPayload = cleanPayload; - } - - public async Task SendPayloadAsync(List payload) - { - return await SendPayloadAsync(payload, true); - } - - public async Task SendPayloadAsync(List payload,bool first) - { - try - { - if (payload == null || !payload.Any()) return new SentPayloadResult(null, true); - var response = await _elasticLowLevelClient.BulkAsync(PostData.MultiJson(payload)); - - if (response.Success) - { - var cleanPayload = new List(); - var invalidPayload = GetInvalidPayloadAsync(response, payload,out cleanPayload); - if ((cleanPayload?.Any() ?? false) && first) - { - await SendPayloadAsync(cleanPayload,false); - } - - return new SentPayloadResult(response, true, invalidPayload); - } - else - { - SelfLog.WriteLine("Received failed ElasticSearch shipping result {0}: {1}", response.HttpStatusCode, - response.OriginalException); - return new SentPayloadResult(response, false, - new InvalidResult() - { - StatusCode = response.HttpStatusCode ?? 500, - Content = response.OriginalException.ToString() - }); - } - } - catch (Exception ex) - { - SelfLog.WriteLine("Exception while emitting periodic batch from {0}: {1}", this, ex); - return new SentPayloadResult(null, false, null, ex); - } - - - } - - private InvalidResult GetInvalidPayloadAsync(DynamicResponse baseResult, List payload, out List cleanPayload) - { - int i = 0; - cleanPayload = new List(); - var items = baseResult.Body["items"]; - if (items == null) return null; - List badPayload = new List(); - - bool hasErrors = false; - foreach (dynamic item in items) - { - long? status = item.index?.status; - i++; - if (!status.HasValue || status < 300) - { - continue; - } - - hasErrors = true; - var id = item.index?._id; - var error = item.index?.error; - if (int.TryParse(id.Split('_')[0], out int index)) - { - SelfLog.WriteLine("Received failed ElasticSearch shipping result {0}: {1}. Failed payload : {2}.", status, error?.ToString(), payload.ElementAt(index * 2 + 1)); - badPayload.Add(payload.ElementAt(index * 2)); - badPayload.Add(payload.ElementAt(index * 2 + 1)); - if (_cleanPayload != null) - { - cleanPayload.Add(payload.ElementAt(index * 2)); - cleanPayload.Add(_cleanPayload(payload.ElementAt(index * 2 + 1), status, error?.ToString())); - } - } - else - { - SelfLog.WriteLine($"Received failed ElasticSearch shipping result {status}: {error?.ToString()}."); - } - } - - if (!hasErrors) - return null; - return new InvalidResult() - { - StatusCode = baseResult.HttpStatusCode ?? 500, - Content = baseResult.ToString(), - BadPayLoad = String.Join(Environment.NewLine, badPayload) - }; - } - } -} diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchLogShipper.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchLogShipper.cs deleted file mode 100644 index 6a6b8b26..00000000 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchLogShipper.cs +++ /dev/null @@ -1,71 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Serilog.Core; -using Serilog.Debugging; -using Serilog.Sinks.Elasticsearch.Durable; - -namespace Serilog.Sinks.Elasticsearch.Durable -{ - /// - /// - /// - public class ElasticsearchLogShipper : LogShipper> - { - private readonly Action _registerTemplateIfNeeded; - bool _didRegisterTemplateIfNeeded = false; - - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - public ElasticsearchLogShipper(string bufferBaseFilename, int batchPostingLimit, TimeSpan period, - long? eventBodyLimitBytes, LoggingLevelSwitch levelControlSwitch, ILogClient> logClient, - IPayloadReader> payloadReader, long? retainedInvalidPayloadsLimitBytes, - long? bufferSizeLimitBytes, Action registerTemplateIfNeeded) - : base(bufferBaseFilename, batchPostingLimit, period, eventBodyLimitBytes, - levelControlSwitch, logClient, payloadReader, retainedInvalidPayloadsLimitBytes, bufferSizeLimitBytes) - { - _registerTemplateIfNeeded = registerTemplateIfNeeded; - } - - /// - /// - /// - /// - protected override async Task OnTick() - { - bool success = true; - try - { - if (!_didRegisterTemplateIfNeeded) - { - if (_registerTemplateIfNeeded != null) - { - _registerTemplateIfNeeded(); - _didRegisterTemplateIfNeeded = true; - } - } - } - catch (Exception ex) - { - SelfLog.WriteLine("Exception while emitting periodic batch from {0}: {1}", this, ex); - _connectionSchedule.MarkFailure(); - success = false; - } - if (success) - await base.OnTick(); - } - } -} diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchPayloadReader.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchPayloadReader.cs deleted file mode 100644 index a31b253e..00000000 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchPayloadReader.cs +++ /dev/null @@ -1,100 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Serilog.Sinks.Elasticsearch.Durable -{ - /// - /// - /// - public class ElasticsearchPayloadReader: APayloadReader> - { - private readonly string _pipelineName; - private readonly string _typeName; - private readonly Func _serialize; - private readonly Func _getIndexForEvent; - private List _payload; - private int _count; - private DateTime _date; - - /// - /// - /// - /// - /// - /// - /// - public ElasticsearchPayloadReader(string pipelineName,string typeName, Func serialize,Func getIndexForEvent) - { - _pipelineName = pipelineName; - _typeName = typeName; - _serialize = serialize; - _getIndexForEvent = getIndexForEvent; - } - - /// - /// - /// - /// - public override List GetNoPayload() - { - return new List(); - } - - /// - /// - /// - /// - protected override void InitPayLoad(string filename) - { - _payload = new List(); - _count = 0; - var lastToken = filename.Split('-').Last(); - - // lastToken should be something like 20150218.json or 20150218_3.json now - if (!lastToken.ToLowerInvariant().EndsWith(".json")) - { - throw new FormatException(string.Format("The file name '{0}' does not seem to follow the right file pattern - it must be named [whatever]-{{Date}}[_n].json", Path.GetFileName(filename))); - } - - var dateString = lastToken.Substring(0, 8); - _date = DateTime.ParseExact(dateString, "yyyyMMdd", CultureInfo.InvariantCulture); - } - /// - /// - /// - /// - protected override List FinishPayLoad() - { - return _payload; - } - - /// - /// - /// - /// - protected override void AddToPayLoad(string nextLine) - { - var indexName = _getIndexForEvent(nextLine, _date); - var action = default(object); - - if (string.IsNullOrWhiteSpace(_pipelineName)) - { - action = new { index = new { _index = indexName, _type = _typeName, _id = _count + "_" + Guid.NewGuid() } }; - } - else - { - action = new { index = new { _index = indexName, _type = _typeName, _id = _count + "_" + Guid.NewGuid(), pipeline = _pipelineName } }; - } - - var actionJson = _serialize(action); - _payload.Add(actionJson); - _payload.Add(nextLine); - _count++; - } - } -} diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/DurableElasticsearchSink.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch/DurableElasticsearchSink.cs similarity index 100% rename from src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/DurableElasticsearchSink.cs rename to src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch/DurableElasticsearchSink.cs diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticsearchLogClient.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch/ElasticsearchLogClient.cs similarity index 100% rename from src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticsearchLogClient.cs rename to src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch/ElasticsearchLogClient.cs diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticsearchLogShipper.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch/ElasticsearchLogShipper.cs similarity index 100% rename from src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticsearchLogShipper.cs rename to src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch/ElasticsearchLogShipper.cs diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticsearchPayloadReader.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch/ElasticsearchPayloadReader.cs similarity index 100% rename from src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticsearchPayloadReader.cs rename to src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch/ElasticsearchPayloadReader.cs diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs index 0bb75390..b00d4d5b 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs @@ -39,6 +39,7 @@ public ElasticsearchSink(ElasticsearchSinkOptions options) : base(options.BatchPostingLimit, options.Period, options.QueueSizeLimit) { _state = ElasticsearchSinkState.Create(options); + _state.DiscoverClusterVersion(); _state.RegisterTemplateIfNeeded(); } @@ -71,7 +72,7 @@ protected override void EmitBatch(IEnumerable events) var items = result.Body["items"]; foreach (var item in items) { - if (item.index != null && item.index.error != null) + if (item["index"] != null && item["index"]["error"] != null) { var e = events.ElementAt(indexer); if (_state.Options.EmitEventFailure.HasFlag(EmitEventFailureHandling.WriteToSelfLog)) @@ -80,8 +81,8 @@ protected override void EmitBatch(IEnumerable events) SelfLog.WriteLine( "Failed to store event with template '{0}' into Elasticsearch. Elasticsearch reports for index {1} the following: {2}", e.MessageTemplate, - item.index._index, - item.index.error); + item["index"]["_index"], + _state.Serialize(item["index"]["error"])); } if (_state.Options.EmitEventFailure.HasFlag(EmitEventFailureHandling.WriteToFailureSink) && diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs index 3847a4b0..6272a563 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using Elasticsearch.Net.Specification.IndicesApi; namespace Serilog.Sinks.Elasticsearch { @@ -19,7 +20,11 @@ public enum AutoRegisterTemplateVersion /// /// Elasticsearch version >= version 6.0 /// - ESv6 = 2 + ESv6 = 2, + /// + /// Elasticsearch version >= version 7.0 + /// + ESv7 = 3 } /// @@ -27,13 +32,7 @@ public enum AutoRegisterTemplateVersion /// public class ElasticsearchTemplateProvider { - /// - /// - /// - /// - /// - /// - /// + [Obsolete("Use the overload taking ElasticsearchSinkOptions which takes IncludeTypeName into account")] public static object GetTemplate( Dictionary settings, string templateMatchString, @@ -44,97 +43,205 @@ public static object GetTemplate( case AutoRegisterTemplateVersion.ESv5: return GetTemplateESv5(settings, templateMatchString); case AutoRegisterTemplateVersion.ESv6: - return GetTemplateESv6(settings, templateMatchString); + return GetTemplateESv6(null, null, settings, templateMatchString); + case AutoRegisterTemplateVersion.ESv7: + return GetTemplateESv7(null, null, settings, templateMatchString); case AutoRegisterTemplateVersion.ESv2: return GetTemplateESv2(settings, templateMatchString); default: throw new ArgumentOutOfRangeException(nameof(version), version, null); } } + + public static object GetTemplate(ElasticsearchSinkOptions options, + string discoveredVersion, + Dictionary settings, + string templateMatchString, + AutoRegisterTemplateVersion version = AutoRegisterTemplateVersion.ESv2) + { + switch (version) + { + case AutoRegisterTemplateVersion.ESv5: + return GetTemplateESv5(settings, templateMatchString); + case AutoRegisterTemplateVersion.ESv2: + return GetTemplateESv2(settings, templateMatchString); + case AutoRegisterTemplateVersion.ESv6: + return GetTemplateESv6(options, discoveredVersion, settings, templateMatchString); + case AutoRegisterTemplateVersion.ESv7: + return GetTemplateESv7(options, discoveredVersion, settings, templateMatchString); + default: + throw new ArgumentOutOfRangeException(nameof(version), version, null); + } + } - private static object GetTemplateESv6( + private static object GetTemplateESv7(ElasticsearchSinkOptions options, string discoveredVersion, Dictionary settings, string templateMatchString) { - return new + object mappings = new { - index_patterns = new[] { templateMatchString }, - settings = settings, - mappings = new + dynamic_templates = new List { - _default_ = new + //when you use serilog as an adaptor for third party frameworks + //where you have no control over the log message they typically + //contain {0} ad infinitum, we force numeric property names to + //contain strings by default. { - dynamic_templates = new List + new { - //when you use serilog as an adaptor for third party frameworks - //where you have no control over the log message they typically - //contain {0} ad infinitum, we force numeric property names to - //contain strings by default. + numerics_in_fields = new { - new + path_match = @"fields\.[\d+]$", + match_pattern = "regex", + mapping = new { - numerics_in_fields = new + type = "text", + index = true, + norms = false + } + } + } + }, + { + new + { + string_fields = new + { + match = "*", + match_mapping_type = "string", + mapping = new + { + type = "text", + index = true, + norms = false, + fields = new { - path_match = @"fields\.[\d+]$", - match_pattern = "regex", - mapping = new + raw = new { - type = "text", + type = "keyword", index = true, - norms = false + ignore_above = 256 } } } - }, + } + } + } + }, + properties = new Dictionary + { + {"message", new {type = "text", index = true}}, + { + "exceptions", new + { + type = "nested", + properties = new Dictionary { - new + {"Depth", new {type = "integer"}}, + {"RemoteStackIndex", new {type = "integer"}}, + {"HResult", new {type = "integer"}}, + {"StackTraceString", new {type = "text", index = true}}, + {"RemoteStackTraceString", new {type = "text", index = true}}, { - string_fields = new + "ExceptionMessage", new { - match = "*", - match_mapping_type = "string", - mapping = new + type = "object", + properties = new Dictionary { - type = "text", + {"MemberType", new {type = "integer"}}, + } + } + } + } + } + } + } + }; + mappings = discoveredVersion?.StartsWith("6.") ?? false ? new { _doc = mappings } : mappings; + + return new + { + index_patterns = new[] {templateMatchString}, + settings = settings, + mappings = mappings + }; + } + + private static object GetTemplateESv6(ElasticsearchSinkOptions options, string discoveredVersion, + Dictionary settings, + string templateMatchString) + { + object mappings = new + { + dynamic_templates = new List + { + //when you use serilog as an adaptor for third party frameworks + //where you have no control over the log message they typically + //contain {0} ad infinitum, we force numeric property names to + //contain strings by default. + { + new + { + numerics_in_fields = new + { + path_match = @"fields\.[\d+]$", + match_pattern = "regex", + mapping = new + { + type = "text", + index = true, + norms = false + } + } + } + }, + { + new + { + string_fields = new + { + match = "*", + match_mapping_type = "string", + mapping = new + { + type = "text", + index = true, + norms = false, + fields = new + { + raw = new + { + type = "keyword", index = true, - norms = false, - fields = new - { - raw = new - { - type = "keyword", - index = true, - ignore_above = 256 - } - } + ignore_above = 256 } } } } - }, - properties = new Dictionary + } + } + }, + properties = new Dictionary + { + {"message", new {type = "text", index = true}}, + { + "exceptions", new { - {"message", new {type = "text", index = "true"}}, + type = "nested", + properties = new Dictionary { - "exceptions", new + {"Depth", new {type = "integer"}}, + {"RemoteStackIndex", new {type = "integer"}}, + {"HResult", new {type = "integer"}}, + {"StackTraceString", new {type = "text", index = true}}, + {"RemoteStackTraceString", new {type = "text", index = true}}, { - type = "nested", - properties = new Dictionary + "ExceptionMessage", new { - {"Depth", new {type = "integer"}}, - {"RemoteStackIndex", new {type = "integer"}}, - {"HResult", new {type = "integer"}}, - {"StackTraceString", new {type = "text", index = "true"}}, - {"RemoteStackTraceString", new {type = "text", index = "true"}}, + type = "object", + properties = new Dictionary { - "ExceptionMessage", new - { - type = "object", - properties = new Dictionary - { - {"MemberType", new {type = "integer"}}, - } - } + {"MemberType", new {type = "integer"}}, } } } @@ -143,6 +250,15 @@ private static object GetTemplateESv6( } } }; + + mappings = discoveredVersion?.StartsWith("7.") ?? false ? (object) new { _doc = mappings} : new { _default_ = mappings}; + + return new + { + index_patterns = new[] { templateMatchString }, + settings = settings, + mappings = mappings + }; } private static object GetTemplateESv5( diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs index 8397d403..0c450796 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs @@ -159,7 +159,7 @@ public class ElasticsearchSinkOptions public IElasticsearchSerializer Serializer { get; set; } /// - /// The connectionpool describing the cluster to write event to + /// The connection pool describing the cluster to write event to /// public IConnectionPool ConnectionPool { get; private set; } @@ -259,7 +259,7 @@ public ElasticsearchSinkOptions() { this.IndexFormat = "logstash-{0:yyyy.MM.dd}"; this.DeadLetterIndexName = "deadletter-{0:yyyy.MM.dd}"; - this.TypeName = "logevent"; + this.TypeName = DefaultTypeName; this.Period = TimeSpan.FromSeconds(2); this.BatchPostingLimit = 50; this.SingleEventSizePostingLimit = null; @@ -271,8 +271,36 @@ public ElasticsearchSinkOptions() this.BufferFileCountLimit = 31; this.BufferFileSizeLimitBytes = 100L * 1024 * 1024; this.FormatStackTraceAsArray = false; + this.ConnectionPool = new SingleNodeConnectionPool(_defaultNode); } + /// + /// The default Elasticsearch type name used for Elasticsearch versions prior to 7. + /// As of Elasticsearch 7 and up _type has been removed. + /// + public static string DefaultTypeName { get; } = "logevent"; + + /// + /// Instructs the sink to auto detect the running Elasticsearch version. + /// + /// + /// This information is used to attempt to register an older or newer template + /// + /// + /// + /// + /// Currently supports: + /// + /// + /// + /// + /// Currently supports: + /// - using against Elasticsearch 6.x + /// - using against Elasticsearch 7.x + /// + /// + public bool DetectElasticsearchVersion { get; set; } + /// /// Configures the elasticsearch sink /// @@ -290,13 +318,13 @@ public ElasticsearchSinkOptions(IConnectionPool connectionPool) public ElasticsearchSinkOptions(IEnumerable nodes) : this() { - nodes = nodes != null && nodes.Any(n => n != null) - ? nodes.Where(n => n != null) - : new[] { new Uri("http://localhost:9200") }; - if (nodes.Count() == 1) - ConnectionPool = new SingleNodeConnectionPool(nodes.First()); + var materialized = nodes?.Where(n => n != null).ToArray(); + if (materialized == null || materialized.Length == 0) + materialized = new[] { _defaultNode }; + if (materialized.Length == 1) + ConnectionPool = new SingleNodeConnectionPool(materialized.First()); else - ConnectionPool = new StaticConnectionPool(nodes); + ConnectionPool = new StaticConnectionPool(materialized); } /// @@ -304,6 +332,8 @@ public ElasticsearchSinkOptions(IEnumerable nodes) /// /// The node to write to public ElasticsearchSinkOptions(Uri node) : this(new[] { node }) { } + + private readonly Uri _defaultNode = new Uri("http://localhost:9200"); } /// diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs index af1a6cbe..7c775632 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs @@ -14,9 +14,12 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Text; using System.Text.RegularExpressions; using Elasticsearch.Net; +using Elasticsearch.Net.Specification.CatApi; +using Elasticsearch.Net.Specification.IndicesApi; using Serilog.Debugging; using Serilog.Events; using Serilog.Formatting; @@ -48,7 +51,12 @@ public static ElasticsearchSinkState Create(ElasticsearchSinkOptions options) private readonly string _templateName; private readonly string _templateMatchString; private static readonly Regex IndexFormatRegex = new Regex(@"^(.*)(?:\{0\:.+\})(.*)$"); + private string _discoveredVersion; + public string DiscoveredVersion => _discoveredVersion; + private bool IncludeTypeName => + (DiscoveredVersion?.StartsWith("7.") ?? false) + && _options.AutoRegisterTemplateVersion == AutoRegisterTemplateVersion.ESv6; public ElasticsearchSinkOptions Options => _options; public IElasticLowLevelClient Client => _client; public ITextFormatter Formatter => _formatter; @@ -59,9 +67,18 @@ public static ElasticsearchSinkState Create(ElasticsearchSinkOptions options) private ElasticsearchSinkState(ElasticsearchSinkOptions options) { if (string.IsNullOrWhiteSpace(options.IndexFormat)) throw new ArgumentException("options.IndexFormat"); - if (string.IsNullOrWhiteSpace(options.TypeName)) throw new ArgumentException("options.TypeName"); if (string.IsNullOrWhiteSpace(options.TemplateName)) throw new ArgumentException("options.TemplateName"); + // Strip type argument if ESv7 since multiple types are not supported anymore + if (options.AutoRegisterTemplateVersion == AutoRegisterTemplateVersion.ESv7) + { + options.TypeName = "_doc"; + } + else + { + if (string.IsNullOrWhiteSpace(options.TypeName)) throw new ArgumentException("options.TypeName"); + } + _templateName = options.TemplateName; _templateMatchString = IndexFormatRegex.Replace(options.IndexFormat, @"$1*$2"); @@ -70,6 +87,7 @@ private ElasticsearchSinkState(ElasticsearchSinkOptions options) _options = options; + var configuration = new ConnectionConfiguration(options.ConnectionPool, options.Connection, options.Serializer) .RequestTimeout(options.ConnectionTimeout); @@ -112,7 +130,7 @@ public static ITextFormatter CreateDefaultDurableFormatter(ElasticsearchSinkOpti public string Serialize(object o) { - return _client.Serializer.SerializeToString(o, SerializationFormatting.None); + return _client.Serializer.SerializeToString(o, formatting: SerializationFormatting.None); } public string GetIndexForEvent(LogEvent e, DateTimeOffset offset) @@ -140,7 +158,10 @@ public void RegisterTemplateIfNeeded() { if (!_options.OverwriteTemplate) { - var templateExistsResponse = _client.IndicesExistsTemplateForAll(_templateName); + var templateExistsResponse = _client.Indices.TemplateExistsForAll(_templateName, new IndexTemplateExistsRequestParameters() + { + RequestConfiguration = new RequestConfiguration() { AllowedStatusCodes = new [] {200, 404} } + }); if (templateExistsResponse.HttpStatusCode == 200) { TemplateRegistrationSuccess = true; @@ -149,7 +170,11 @@ public void RegisterTemplateIfNeeded() } } - var result = _client.IndicesPutTemplateForAll(_templateName, GetTempatePostData()); + var result = _client.Indices.PutTemplateForAll(_templateName, GetTemplatePostData(), + new PutIndexTemplateRequestParameters + { + IncludeTypeName = IncludeTypeName ? true : (bool?) null + }); if (!result.Success) { @@ -176,7 +201,7 @@ public void RegisterTemplateIfNeeded() } } - private PostData GetTempatePostData() + private PostData GetTemplatePostData() { //PostData no longer exposes an implict cast from object. Previously it supported that and would inspect the object Type to //determine if it it was a litteral string to write directly or if it was an object that it needed to serialse. Now the onus is @@ -209,10 +234,29 @@ private object GetTemplateData() settings.Add("number_of_replicas", _options.NumberOfReplicas.Value.ToString()); return ElasticsearchTemplateProvider.GetTemplate( + _options, + DiscoveredVersion, settings, _templateMatchString, _options.AutoRegisterTemplateVersion); } + + public void DiscoverClusterVersion() + { + if (!_options.DetectElasticsearchVersion) return; + + var response = _client.Cat.Nodes(new CatNodesRequestParameters() + { + Headers = new [] { "v"} + }); + if (!response.Success) return; + + _discoveredVersion = response.Body.Split(new[] {'\r', '\n'}, StringSplitOptions.RemoveEmptyEntries) + .FirstOrDefault(); + + if (_discoveredVersion?.StartsWith("7.") ?? false) + _options.TypeName = "_doc"; + } } } diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/SerializerAdapter.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/SerializerAdapter.cs index c551df02..c171a79b 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/SerializerAdapter.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/SerializerAdapter.cs @@ -53,7 +53,7 @@ public Task SerializeAsync(T data, Stream stream, public string SerializeToString(object value) { - return _elasticsearchSerializer.SerializeToString(value, SerializationFormatting.None); + return _elasticsearchSerializer.SerializeToString(value, formatting: SerializationFormatting.None); } } } \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ClientTestClusterBase.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ClientTestClusterBase.cs new file mode 100644 index 00000000..313b6f3a --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ClientTestClusterBase.cs @@ -0,0 +1,37 @@ +using Elastic.Managed.Ephemeral; +using Elastic.Managed.Ephemeral.Plugins; +using Elastic.Xunit; +using Nest; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap +{ + public abstract class ClientTestClusterBase : XunitClusterBase + { + protected ClientTestClusterBase(ClientTestClusterConfiguration configuration) : base(configuration) { } + } + + public class ClientTestClusterConfiguration : XunitClusterConfiguration + { + public ClientTestClusterConfiguration( + string elasticsearchVersion, + ClusterFeatures features = ClusterFeatures.None, + int numberOfNodes = 1, + params ElasticsearchPlugin[] plugins + ) + : base(elasticsearchVersion, features, new ElasticsearchPlugins(plugins), numberOfNodes) + { + HttpFiddlerAware = true; + CacheEsHomeInstallation = true; + + Add(AttributeKey("testingcluster"), "true"); + + Add($"script.max_compilations_per_minute", "10000", "<6.0.0-rc1"); + Add($"script.max_compilations_rate", "10000/1m", ">=6.0.0-rc1"); + + Add($"script.inline", "true", "<5.5.0"); + Add($"script.stored", "true", ">5.0.0-alpha1 <5.5.0"); + Add($"script.indexed", "true", "<5.0.0-alpha1"); + Add($"script.allowed_types", "inline,stored", ">=5.5.0"); + } + } +} diff --git a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ElasticsearchSinkOptionsFactory.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ElasticsearchSinkOptionsFactory.cs new file mode 100644 index 00000000..27279840 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ElasticsearchSinkOptionsFactory.cs @@ -0,0 +1,36 @@ +using System; +using Elasticsearch.Net; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap +{ + public static class ElasticsearchSinkOptionsFactory + { + public static ElasticsearchSinkOptions Create(string indexPrefix, string templateName, Action alterOptions = null) + { + // make sure we run through `ipv4.fiddler` if `fiddler` or `mitmproxy` is running + // NOTE with the latter you need to add `ipv4.fiddler` as an alias to 127.0.0.1 in your HOSTS file manually + var pool = new SingleNodeConnectionPool(new Uri($"http://{ProxyDetection.LocalOrProxyHost}:9200")); + var options = new ElasticsearchSinkOptions(pool) + { + IndexFormat = indexPrefix + "{0:yyyy.MM.dd}", + TemplateName = templateName, + }; + + alterOptions?.Invoke(options); + // here we make sure we set a proxy on the elasticsearch connection + // when we detect `mitmproxy` running. This is a cli tool similar to fiddler + // on *nix systems which aids debugging what goes over the wire + var provided = options.ModifyConnectionSettings; + options.ModifyConnectionSettings = configuration => + { + if (ProxyDetection.RunningMitmProxy) + configuration.Proxy(ProxyDetection.MitmProxyAddress, null, (string) null); + configuration = provided?.Invoke(configuration) ?? configuration; + return configuration; + }; + + return options; + } + + } +} \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ProxyDetection.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ProxyDetection.cs new file mode 100644 index 00000000..0ba27742 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ProxyDetection.cs @@ -0,0 +1,15 @@ +using System; +using System.Diagnostics; +using System.Linq; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap +{ + public static class ProxyDetection + { + public static readonly bool RunningMitmProxy = Process.GetProcessesByName("mitmproxy").Any(); + private static readonly bool RunningFiddler = Process.GetProcessesByName("fiddler").Any(); + public static string LocalOrProxyHost => RunningFiddler || RunningMitmProxy ? "ipv4.fiddler" : "localhost"; + + public static readonly Uri MitmProxyAddress = new Uri("http://localhost:8080"); + } +} \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/SerilogSinkElasticsearchXunitRunOptions.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/SerilogSinkElasticsearchXunitRunOptions.cs new file mode 100644 index 00000000..ea48e4c4 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/SerilogSinkElasticsearchXunitRunOptions.cs @@ -0,0 +1,107 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using Elastic.Xunit; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap +{ + /// Feeding TestClient.Configuration options to the runner + public class SerilogSinkElasticsearchXunitRunOptions : ElasticXunitRunOptions + { + public SerilogSinkElasticsearchXunitRunOptions() + { + RunIntegrationTests = true; + RunUnitTests = false; + ClusterFilter = null; + TestFilter = null; + IntegrationTestsMayUseAlreadyRunningNode = false; + } + + public override void OnBeforeTestsRun() { } + + public override void OnTestsFinished(Dictionary clusterTotals, ConcurrentBag> failedCollections) + { + Console.Out.Flush(); + DumpClusterTotals(clusterTotals); + DumpFailedCollections(failedCollections); + } + + private static void DumpClusterTotals(Dictionary clusterTotals) + { + Console.WriteLine("--------"); + Console.WriteLine("Individual cluster running times:"); + foreach (var kv in clusterTotals) Console.WriteLine($"- {kv.Key}: {kv.Value.Elapsed}"); + Console.WriteLine("--------"); + } + + private static void DumpFailedCollections(ConcurrentBag> failedCollections) + { + if (failedCollections.Count <= 0) return; + + Console.ForegroundColor = ConsoleColor.Red; + Console.WriteLine("Failed collections:"); + foreach (var t in failedCollections.OrderBy(p => p.Item1).ThenBy(t => t.Item2)) + + { + var cluster = t.Item1; + Console.WriteLine($" - {cluster}: {t.Item2}"); + } + DumpReproduceFilters(failedCollections); + Console.ResetColor(); + } + + private static void DumpReproduceFilters(ConcurrentBag> failedCollections) + { + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("---Reproduce: -----"); + var reproduceLine = ReproduceCommandLine(failedCollections); + Console.WriteLine(reproduceLine); + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TEAMCITY_VERSION"))) + Console.WriteLine($"##teamcity[buildProblem description='{reproduceLine}']"); + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TF_BUILD"))) + { + var count = failedCollections.Count; + Console.WriteLine($"##vso[task.logissue type=error;]{count} test failures"); + Console.WriteLine($"##vso[task.logissue type=error;]{reproduceLine}"); + } + Console.WriteLine("--------"); + } + + private static string ReproduceCommandLine(ConcurrentBag> failedCollections) + { + var sb = new StringBuilder("build.bat "); + + if (failedCollections.Count > 0) + { + var clusters = string.Join(",", failedCollections + .Select(c => c.Item1.ToLowerInvariant()) + .Distinct()); + sb.Append(" \""); + sb.Append(clusters); + sb.Append("\""); + } + + if ((failedCollections.Count < 30) && failedCollections.Count > 0) + { + sb.Append(" \""); + var tests = string.Join(",", failedCollections + .OrderBy(t => t.Item2) + .Select(c => c.Item2.ToLowerInvariant() + .Split('.') + .Last() + .Replace("apitests", "") + .Replace("usagetests", "") + .Replace("tests", "") + )); + sb.Append(tests); + sb.Append("\""); + } + + var reproduceLine = sb.ToString(); + return reproduceLine; + } + } +} diff --git a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/XunitBootstrap.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/XunitBootstrap.cs new file mode 100644 index 00000000..65de3e39 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/XunitBootstrap.cs @@ -0,0 +1,6 @@ +using Elastic.Xunit; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap; +using Xunit; + +[assembly: TestFramework("Elastic.Xunit.Sdk.ElasticTestFramework", "Elastic.Xunit")] +[assembly: ElasticXunitConfiguration(typeof(SerilogSinkElasticsearchXunitRunOptions))] diff --git a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Bootstrap/Elasticsearch6XCluster.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Bootstrap/Elasticsearch6XCluster.cs new file mode 100644 index 00000000..30d6d229 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Bootstrap/Elasticsearch6XCluster.cs @@ -0,0 +1,19 @@ +using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch6.Bootstrap +{ + public class Elasticsearch6XCluster : ClientTestClusterBase + { + public Elasticsearch6XCluster() : base(CreateConfiguration()) { } + + private static ClientTestClusterConfiguration CreateConfiguration() + { + return new ClientTestClusterConfiguration("6.6.0") + { + MaxConcurrency = 1 + }; + } + + protected override void SeedCluster() { } + } +} diff --git a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Bootstrap/Elasticsearch6XTestBase.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Bootstrap/Elasticsearch6XTestBase.cs new file mode 100644 index 00000000..bc4f95d9 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Bootstrap/Elasticsearch6XTestBase.cs @@ -0,0 +1,50 @@ +using System; +using System.Security.Cryptography.X509Certificates; +using Elastic.Managed.Ephemeral; +using Elastic.Xunit; +using Elastic.Xunit.XunitPlumbing; +using Elasticsearch.Net6; +using Nest6; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch6.Bootstrap +{ + public abstract class Elasticsearch6XTestBase : IClusterFixture + { + protected Elasticsearch6XTestBase(Elasticsearch6XCluster cluster) => Cluster = cluster; + + private Elasticsearch6XCluster Cluster { get; } + + protected IElasticClient Client => this.CreateClient(); + + protected virtual ConnectionSettings SetClusterSettings(ConnectionSettings s) => s; + + private IElasticClient CreateClient() => + Cluster.GetOrAddClient(c => + { + var clusterNodes = c.NodesUris(ProxyDetection.LocalOrProxyHost); + var settings = new ConnectionSettings(new StaticConnectionPool(clusterNodes)); + if (ProxyDetection.RunningMitmProxy) + settings = settings.Proxy(new Uri("http://localhost:8080"), null, null); + settings = SetClusterSettings(settings); + + var current = (IConnectionConfigurationValues)settings; + var notAlreadyAuthenticated = current.BasicAuthenticationCredentials == null && current.ClientCertificates == null; + var noCertValidation = current.ServerCertificateValidationCallback == null; + + var config = Cluster.ClusterConfiguration; + if (config.EnableSecurity && notAlreadyAuthenticated) + settings = settings.BasicAuthentication(ClusterAuthentication.Admin.Username, ClusterAuthentication.Admin.Password); + if (config.EnableSsl && noCertValidation) + { + //todo use CA callback instead of allow all + // ReSharper disable once UnusedVariable + var ca = new X509Certificate2(config.FileSystem.CaCertificate); + settings = settings.ServerCertificateValidationCallback(CertificateValidations.AllowAll); + } + var client = new ElasticClient(settings); + return client; + }); + } + +} diff --git a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6X.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6X.cs new file mode 100644 index 00000000..9bb47c2f --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6X.cs @@ -0,0 +1,71 @@ +using System.Linq; +using Elastic.Xunit.XunitPlumbing; +using FluentAssertions; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch6.Bootstrap; +using Xunit; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch6 +{ + public class Elasticsearch6X : Elasticsearch6XTestBase, IClassFixture + { + private readonly SetupSerilog _setup; + + public Elasticsearch6X(Elasticsearch6XCluster cluster, SetupSerilog setup) : base(cluster) => _setup = setup; + + + [I] public void AssertTemplate() + { + var templateResponse = Client.GetIndexTemplate(t=>t.Name(SetupSerilog.TemplateName)); + templateResponse.TemplateMappings.Should().NotBeEmpty(); + templateResponse.TemplateMappings.Keys.Should().Contain(SetupSerilog.TemplateName); + + var template = templateResponse.TemplateMappings[SetupSerilog.TemplateName]; + + template.IndexPatterns.Should().Contain(pattern => pattern.StartsWith(SetupSerilog.IndexPrefix)); + } + + [I] public void AssertLogs() + { + var refreshed = Client.Refresh(SetupSerilog.IndexPrefix + "*"); + + var search = Client.Search(s => s + .Index(SetupSerilog.IndexPrefix + "*") + .Type(ElasticsearchSinkOptions.DefaultTypeName) + ); + + // Informational should be filtered + search.Documents.Count().Should().Be(4); + } + + // ReSharper disable once ClassNeverInstantiated.Global + public class SetupSerilog + { + public const string IndexPrefix = "logs-6x-default-"; + public const string TemplateName = "serilog-logs-6x"; + + public SetupSerilog() + { + var loggerConfig = new LoggerConfiguration() + .MinimumLevel.Information() + .WriteTo.ColoredConsole() + .WriteTo.Elasticsearch( + ElasticsearchSinkOptionsFactory.Create(IndexPrefix, TemplateName, o => + { + o.AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv6; + o.AutoRegisterTemplate = true; + }) + ); + var logger = loggerConfig.CreateLogger(); + + logger.Information("Hello Information"); + logger.Debug("Hello Debug"); + logger.Warning("Hello Warning"); + logger.Error("Hello Error"); + logger.Fatal("Hello Fatal"); + + logger.Dispose(); + } + } + } +} \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6XUsing7X.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6XUsing7X.cs new file mode 100644 index 00000000..9078c9c8 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6XUsing7X.cs @@ -0,0 +1,71 @@ +using System.Linq; +using Elastic.Xunit.XunitPlumbing; +using FluentAssertions; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch6.Bootstrap; +using Xunit; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch6 +{ + public class Elasticsearch6XUsing7X : Elasticsearch6XTestBase, IClassFixture + { + private readonly SetupSerilog _setup; + + public Elasticsearch6XUsing7X(Elasticsearch6XCluster cluster, SetupSerilog setup) : base(cluster) => _setup = setup; + + [I] public void AssertTemplate() + { + var templateResponse = Client.GetIndexTemplate(t=>t.Name(SetupSerilog.TemplateName)); + templateResponse.TemplateMappings.Should().NotBeEmpty(); + templateResponse.TemplateMappings.Keys.Should().Contain(SetupSerilog.TemplateName); + + var template = templateResponse.TemplateMappings[SetupSerilog.TemplateName]; + + template.IndexPatterns.Should().Contain(pattern => pattern.StartsWith(SetupSerilog.IndexPrefix)); + } + + [I] public void AssertLogs() + { + var refreshed = Client.Refresh(SetupSerilog.IndexPrefix + "*"); + + var search = Client.Search(s => s + .Index(SetupSerilog.IndexPrefix + "*") + .Type("_doc") + ); + + // Informational should be filtered + search.Documents.Count().Should().Be(4); + } + + // ReSharper disable once ClassNeverInstantiated.Global + public class SetupSerilog + { + public const string IndexPrefix = "logs-6x-using-7x-"; + public const string TemplateName = "serilog-logs-6x-using-7x"; + + public SetupSerilog() + { + var loggerConfig = new LoggerConfiguration() + .MinimumLevel.Information() + .WriteTo.ColoredConsole() + .WriteTo.Elasticsearch( + ElasticsearchSinkOptionsFactory.Create(IndexPrefix, TemplateName, o => + { + o.DetectElasticsearchVersion = true; + o.AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7; + o.AutoRegisterTemplate = true; + }) + ); + var logger = loggerConfig.CreateLogger(); + + logger.Information("Hello Information"); + logger.Debug("Hello Debug"); + logger.Warning("Hello Warning"); + logger.Error("Hello Error"); + logger.Fatal("Hello Fatal"); + + logger.Dispose(); + } + } + } +} \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Bootstrap/Elasticsearch7XCluster.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Bootstrap/Elasticsearch7XCluster.cs new file mode 100644 index 00000000..b1f2ab50 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Bootstrap/Elasticsearch7XCluster.cs @@ -0,0 +1,19 @@ +using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch7.Bootstrap +{ + public class Elasticsearch7XCluster : ClientTestClusterBase + { + public Elasticsearch7XCluster() : base(CreateConfiguration()) { } + + private static ClientTestClusterConfiguration CreateConfiguration() + { + return new ClientTestClusterConfiguration("7.0.0") + { + MaxConcurrency = 1 + }; + } + + protected override void SeedCluster() { } + } +} diff --git a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Bootstrap/Elasticsearch7XTestBase.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Bootstrap/Elasticsearch7XTestBase.cs new file mode 100644 index 00000000..281c88fd --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Bootstrap/Elasticsearch7XTestBase.cs @@ -0,0 +1,51 @@ +using System; +using System.Security.Cryptography.X509Certificates; +using Elastic.Managed.Ephemeral; +using Elastic.Xunit; +using Elastic.Xunit.XunitPlumbing; +using Elasticsearch.Net; +using Nest; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap; +using Xunit; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch7.Bootstrap +{ + public abstract class Elasticsearch7XTestBase : IClusterFixture + { + protected Elasticsearch7XTestBase(Elasticsearch7XCluster cluster) => Cluster = cluster; + + private Elasticsearch7XCluster Cluster { get; } + + protected IElasticClient Client => this.CreateClient(); + + protected virtual ConnectionSettings SetClusterSettings(ConnectionSettings s) => s; + + private IElasticClient CreateClient() => + Cluster.GetOrAddClient(c => + { + var clusterNodes = c.NodesUris(ProxyDetection.LocalOrProxyHost); + var settings = new ConnectionSettings(new StaticConnectionPool(clusterNodes)); + if (ProxyDetection.RunningMitmProxy) + settings = settings.Proxy(new Uri("http://localhost:8080"), null, (string)null); + settings = SetClusterSettings(settings); + + var current = (IConnectionConfigurationValues)settings; + var notAlreadyAuthenticated = current.BasicAuthenticationCredentials == null && current.ClientCertificates == null; + var noCertValidation = current.ServerCertificateValidationCallback == null; + + var config = Cluster.ClusterConfiguration; + if (config.EnableSecurity && notAlreadyAuthenticated) + settings = settings.BasicAuthentication(ClusterAuthentication.Admin.Username, ClusterAuthentication.Admin.Password); + if (config.EnableSsl && noCertValidation) + { + //todo use CA callback instead of allow all + // ReSharper disable once UnusedVariable + var ca = new X509Certificate2(config.FileSystem.CaCertificate); + settings = settings.ServerCertificateValidationCallback(CertificateValidations.AllowAll); + } + var client = new ElasticClient(settings); + return client; + }); + } + +} diff --git a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7X.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7X.cs new file mode 100644 index 00000000..a3d19d70 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7X.cs @@ -0,0 +1,67 @@ +using System.Linq; +using Elastic.Xunit.XunitPlumbing; +using FluentAssertions; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch7.Bootstrap; +using Xunit; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch7 +{ + public class Elasticsearch7X : Elasticsearch7XTestBase, IClassFixture + { + private readonly SetupSerilog _setup; + + public Elasticsearch7X(Elasticsearch7XCluster cluster, SetupSerilog setup) : base(cluster) => _setup = setup; + + [I] public void AssertTemplate() + { + var templateResponse = Client.Indices.GetTemplate(SetupSerilog.TemplateName); + templateResponse.TemplateMappings.Should().NotBeEmpty(); + templateResponse.TemplateMappings.Keys.Should().Contain(SetupSerilog.TemplateName); + + var template = templateResponse.TemplateMappings[SetupSerilog.TemplateName]; + + template.IndexPatterns.Should().Contain(pattern => pattern.StartsWith(SetupSerilog.IndexPrefix)); + } + + [I] public void AssertLogs() + { + var refreshed = Client.Indices.Refresh(SetupSerilog.IndexPrefix + "*"); + + var search = Client.Search(s => s.Index(SetupSerilog.IndexPrefix + "*")); + + // Informational should be filtered + search.Documents.Count().Should().Be(4); + } + + // ReSharper disable once ClassNeverInstantiated.Global + public class SetupSerilog + { + public const string IndexPrefix = "logs-7x-default-"; + public const string TemplateName = "serilog-logs-7x"; + + public SetupSerilog() + { + var loggerConfig = new LoggerConfiguration() + .MinimumLevel.Information() + .WriteTo.ColoredConsole() + .WriteTo.Elasticsearch( + ElasticsearchSinkOptionsFactory.Create(IndexPrefix, TemplateName, o => + { + o.AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7; + o.AutoRegisterTemplate = true; + }) + ); + using (var logger = loggerConfig.CreateLogger()) + { + logger.Information("Hello Information"); + logger.Debug("Hello Debug"); + logger.Warning("Hello Warning"); + logger.Error("Hello Error"); + logger.Fatal("Hello Fatal"); + } + } + } + + } +} \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7XUsing6X.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7XUsing6X.cs new file mode 100644 index 00000000..69673882 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7XUsing6X.cs @@ -0,0 +1,68 @@ +using System.Linq; +using Elastic.Xunit.XunitPlumbing; +using FluentAssertions; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch7.Bootstrap; +using Xunit; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch7 +{ + public class Elasticsearch7XUsing6X : Elasticsearch7XTestBase, IClassFixture + { + private readonly SetupSerilog _setup; + + public Elasticsearch7XUsing6X(Elasticsearch7XCluster cluster, SetupSerilog setup) : base(cluster) => _setup = setup; + + [I] public void AssertTemplate() + { + var templateResponse = Client.Indices.GetTemplate(SetupSerilog.TemplateName); + templateResponse.TemplateMappings.Should().NotBeEmpty(); + templateResponse.TemplateMappings.Keys.Should().Contain(SetupSerilog.TemplateName); + + var template = templateResponse.TemplateMappings[SetupSerilog.TemplateName]; + + template.IndexPatterns.Should().Contain(pattern => pattern.StartsWith(SetupSerilog.IndexPrefix)); + } + + [I] public void AssertLogs() + { + var refreshed = Client.Indices.Refresh(SetupSerilog.IndexPrefix + "*"); + + var search = Client.Search(s => s.Index(SetupSerilog.IndexPrefix + "*")); + + // Informational should be filtered + search.Documents.Count().Should().Be(4); + } + + // ReSharper disable once ClassNeverInstantiated.Global + public class SetupSerilog + { + public const string IndexPrefix = "logs-7x-using-6x"; + public const string TemplateName = "serilog-logs-7x-using-6x"; + + public SetupSerilog() + { + var loggerConfig = new LoggerConfiguration() + .MinimumLevel.Information() + .WriteTo.ColoredConsole() + .WriteTo.Elasticsearch( + ElasticsearchSinkOptionsFactory.Create(IndexPrefix, TemplateName, o => + { + o.DetectElasticsearchVersion = true; + o.AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv6; + o.AutoRegisterTemplate = true; + }) + ); + using (var logger = loggerConfig.CreateLogger()) + { + logger.Information("Hello Information"); + logger.Debug("Hello Debug"); + logger.Warning("Hello Warning"); + logger.Error("Hello Error"); + logger.Fatal("Hello Fatal"); + } + } + } + + } +} \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj new file mode 100644 index 00000000..75df89c8 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj @@ -0,0 +1,28 @@ + + + + netcoreapp2.1;net461 + $(NoWarn);xUnit1013 + True + latest + True + + + + + + + + + + + + + + + + + + + + diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs b/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs index eb97ad66..e42f11c3 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs @@ -12,6 +12,7 @@ using Serilog.Sinks.Elasticsearch.Tests.Domain; using Nest.JsonNetSerializer; using System.Collections; +using System.Threading; namespace Serilog.Sinks.Elasticsearch.Tests { @@ -135,7 +136,7 @@ Func templateExistReturnCode public override TReturn Request(RequestData requestData) { - MemoryStream ms = new MemoryStream(); + var ms = new MemoryStream(); if (requestData.PostData != null) requestData.PostData.Write(ms, new ConnectionConfiguration()); @@ -155,6 +156,29 @@ public override TReturn Request(RequestData requestData) var responseStream = new MemoryStream(); return ResponseBuilder.ToResponse(requestData, null, this._templateExistReturnCode(), Enumerable.Empty(), responseStream); } + + public override async Task RequestAsync(RequestData requestData, CancellationToken cancellationToken) + { + var ms = new MemoryStream(); + if (requestData.PostData != null) + await requestData.PostData.WriteAsync(ms, new ConnectionConfiguration(), cancellationToken); + + switch (requestData.Method) + { + case HttpMethod.PUT: + _seenHttpPuts.Add(Tuple.Create(requestData.Uri, Encoding.UTF8.GetString(ms.ToArray()))); + break; + case HttpMethod.POST: + _seenHttpPosts.Add(Encoding.UTF8.GetString(ms.ToArray())); + break; + case HttpMethod.HEAD: + _seenHttpHeads.Add(this._templateExistReturnCode()); + break; + } + + var responseStream = new MemoryStream(); + return await ResponseBuilder.ToResponseAsync(requestData, null, this._templateExistReturnCode(), Enumerable.Empty(), responseStream, null, cancellationToken); + } } } } \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj b/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj index 03f8600f..a1f18554 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj @@ -1,12 +1,15 @@  - netcoreapp1.1;netcoreapp2.0;net451 + netcoreapp2.1;net461 Serilog.Sinks.Elasticsearch.Tests Serilog.Sinks.Elasticsearch.Tests + + True + latest + True + true - $(PackageTargetFallback);dnxcore50;portable-net45+win8 - 1.1.2 false false false @@ -24,9 +27,11 @@ + + @@ -39,15 +44,16 @@ - + + - - - + + + @@ -63,18 +69,15 @@ - - $(DefineConstants);DOTNETCORE;NO_SERIALIZATION - - + + + + + $(DefineConstants);DOTNETCORE;PARTIALLY_SERIALIZATION - - - - diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SendsTemplateHandlesUnavailableServerTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SendsTemplateHandlesUnavailableServerTests.cs index 76f2e0aa..d549f544 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SendsTemplateHandlesUnavailableServerTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SendsTemplateHandlesUnavailableServerTests.cs @@ -34,7 +34,7 @@ public void Should_write_error_to_self_log() private static ILogger CreateLoggerThatCrashes() { var loggerConfig = new LoggerConfiguration() - .WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri("http://localhost:31234")) + .WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri("http://localhost:9199")) { AutoRegisterTemplate = true, TemplateName = "crash" diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv7TemplateTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv7TemplateTests.cs new file mode 100644 index 00000000..6180ed44 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv7TemplateTests.cs @@ -0,0 +1,66 @@ +using System; +using System.Reflection; +using FluentAssertions; +using Newtonsoft.Json.Linq; +using Xunit; + +namespace Serilog.Sinks.Elasticsearch.Tests.Templating +{ + public class Sendsv7TemplateTests : ElasticsearchSinkTestsBase + { + private readonly Tuple _templatePut; + + public Sendsv7TemplateTests() + { + _options.AutoRegisterTemplate = true; + _options.AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7; + + var loggerConfig = new LoggerConfiguration() + .MinimumLevel.Debug() + .Enrich.WithMachineName() + .WriteTo.ColoredConsole() + .WriteTo.Elasticsearch(_options); + + var logger = loggerConfig.CreateLogger(); + using (logger as IDisposable) + { + logger.Error("Test exception. Should not contain an embedded exception object."); + } + + this._seenHttpPosts.Should().NotBeNullOrEmpty().And.HaveCount(1); + this._seenHttpPuts.Should().NotBeNullOrEmpty().And.HaveCount(1); + _templatePut = this._seenHttpPuts[0]; + } + + [Fact] + public void ShouldRegisterTheCorrectTemplateOnRegistration() + { + + var method = typeof(Sendsv7TemplateTests).GetMethod(nameof(ShouldRegisterTheCorrectTemplateOnRegistration)); + JsonEquals(_templatePut.Item2, method, "template_v7.json"); + } + + [Fact] + public void TemplatePutToCorrectUrl() + { + var uri = _templatePut.Item1; + uri.AbsolutePath.Should().Be("/_template/serilog-events-template"); + } + + protected void JsonEquals(string json, MethodBase method, string fileName = null) + { +#if DOTNETCORE + var assembly = typeof(Sendsv7TemplateTests).GetTypeInfo().Assembly; +#else + var assembly = Assembly.GetExecutingAssembly(); +#endif + var expected = TestDataHelper.ReadEmbeddedResource(assembly, fileName ?? "template.json"); + + var nJson = JObject.Parse(json); + var nOtherJson = JObject.Parse(expected); + var equals = JToken.DeepEquals(nJson, nOtherJson); + if (equals) return; + expected.Should().BeEquivalentTo(json); + } + } +} \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v6.json b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v6.json index 382f100e..582c2e9b 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v6.json +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v6.json @@ -39,7 +39,7 @@ "properties": { "message": { "type": "text", - "index": "true" + "index": true }, "exceptions": { "type": "nested", @@ -55,11 +55,11 @@ }, "StackTraceString": { "type": "text", - "index": "true" + "index": true }, "RemoteStackTraceString": { "type": "text", - "index": "true" + "index": true }, "ExceptionMessage": { "type": "object", diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v7.json b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v7.json new file mode 100644 index 00000000..003ada40 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v7.json @@ -0,0 +1,75 @@ +{ + "index_patterns": [ "logstash-*" ], + "settings": { + "index.refresh_interval": "5s" + }, + "mappings": { + "dynamic_templates": [ + { + "numerics_in_fields": { + "path_match": "fields\\.[\\d+]$", + "match_pattern": "regex", + "mapping": { + "type": "text", + "index": true, + "norms": false + } + } + }, + { + "string_fields": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "text", + "index": true, + "norms": false, + "fields": { + "raw": { + "type": "keyword", + "index": true, + "ignore_above": 256 + } + } + } + } + } + ], + "properties": { + "message": { + "type": "text", + "index": true + }, + "exceptions": { + "type": "nested", + "properties": { + "Depth": { + "type": "integer" + }, + "RemoteStackIndex": { + "type": "integer" + }, + "HResult": { + "type": "integer" + }, + "StackTraceString": { + "type": "text", + "index": true + }, + "RemoteStackTraceString": { + "type": "text", + "index": true + }, + "ExceptionMessage": { + "type": "object", + "properties": { + "MemberType": { + "type": "integer" + } + } + } + } + } + } + } +}