From 549f79104886112d57186678001009b616eb796b Mon Sep 17 00:00:00 2001 From: Daniel Rusznyak Date: Wed, 29 May 2019 17:02:33 +0100 Subject: [PATCH 01/15] Serilog.Skinks.Elasticsearch with V7 compatibility --- global.json | 2 +- .../Serilog.Sinks.Elasticsearch.csproj | 10 +- .../Sinks/ElasticSearch/ElasticSearchSink.cs | 6 +- .../ElasticSearchTemplateProvider.cs | 98 ++++++++++++++++++- .../ElasticSearch/ElasticsearchSinkState.cs | 15 ++- .../Sinks/ElasticSearch/SerializerAdapter.cs | 2 +- .../Serilog.Sinks.Elasticsearch.Tests.csproj | 14 ++- .../Templating/template_v7.json | 75 ++++++++++++++ 8 files changed, 212 insertions(+), 10 deletions(-) create mode 100644 test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v7.json diff --git a/global.json b/global.json index 341f9fea..99e7444e 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "2.1.4" + "version": "2.1.500" } } \ No newline at end of file diff --git a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj index b1d79191..d2e3559f 100644 --- a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj +++ b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj @@ -33,7 +33,6 @@ - @@ -43,7 +42,16 @@ + + + + + + + + + 1591;1701;1702 diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs index 0bb75390..d89136fd 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs @@ -71,7 +71,7 @@ protected override void EmitBatch(IEnumerable events) var items = result.Body["items"]; foreach (var item in items) { - if (item.index != null && item.index.error != null) + if (item["index"] != null && item["index"]["error"] != null) { var e = events.ElementAt(indexer); if (_state.Options.EmitEventFailure.HasFlag(EmitEventFailureHandling.WriteToSelfLog)) @@ -80,8 +80,8 @@ protected override void EmitBatch(IEnumerable events) SelfLog.WriteLine( "Failed to store event with template '{0}' into Elasticsearch. Elasticsearch reports for index {1} the following: {2}", e.MessageTemplate, - item.index._index, - item.index.error); + item["index"]["_index"], + _state.Serialize(item["index"]["error"])); } if (_state.Options.EmitEventFailure.HasFlag(EmitEventFailureHandling.WriteToFailureSink) && diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs index 3847a4b0..0325992b 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs @@ -19,7 +19,11 @@ public enum AutoRegisterTemplateVersion /// /// Elasticsearch version >= version 6.0 /// - ESv6 = 2 + ESv6 = 2, + /// + /// Elasticsearch version >= version 7.0 + /// + ESv7 = 3 } /// @@ -45,6 +49,8 @@ public static object GetTemplate( return GetTemplateESv5(settings, templateMatchString); case AutoRegisterTemplateVersion.ESv6: return GetTemplateESv6(settings, templateMatchString); + case AutoRegisterTemplateVersion.ESv7: + return GetTemplateESv7(settings, templateMatchString); case AutoRegisterTemplateVersion.ESv2: return GetTemplateESv2(settings, templateMatchString); default: @@ -52,6 +58,96 @@ public static object GetTemplate( } } + private static object GetTemplateESv7( + Dictionary settings, + string templateMatchString) + { + return new + { + index_patterns = new[] { templateMatchString }, + settings = settings, + mappings = new + { + dynamic_templates = new List + { + //when you use serilog as an adaptor for third party frameworks + //where you have no control over the log message they typically + //contain {0} ad infinitum, we force numeric property names to + //contain strings by default. + { + new + { + numerics_in_fields = new + { + path_match = @"fields\.[\d+]$", + match_pattern = "regex", + mapping = new + { + type = "text", + index = true, + norms = false + } + } + } + }, + { + new + { + string_fields = new + { + match = "*", + match_mapping_type = "string", + mapping = new + { + type = "text", + index = true, + norms = false, + fields = new + { + raw = new + { + type = "keyword", + index = true, + ignore_above = 256 + } + } + } + } + } + } + }, + properties = new Dictionary + { + {"message", new {type = "text", index = "true"}}, + { + "exceptions", new + { + type = "nested", + properties = new Dictionary + { + {"Depth", new {type = "integer"}}, + {"RemoteStackIndex", new {type = "integer"}}, + {"HResult", new {type = "integer"}}, + {"StackTraceString", new {type = "text", index = "true"}}, + {"RemoteStackTraceString", new {type = "text", index = "true"}}, + { + "ExceptionMessage", new + { + type = "object", + properties = new Dictionary + { + {"MemberType", new {type = "integer"}}, + } + } + } + } + } + } + } + } + }; + } + private static object GetTemplateESv6( Dictionary settings, string templateMatchString) diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs index af1a6cbe..679857c9 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs @@ -59,9 +59,18 @@ public static ElasticsearchSinkState Create(ElasticsearchSinkOptions options) private ElasticsearchSinkState(ElasticsearchSinkOptions options) { if (string.IsNullOrWhiteSpace(options.IndexFormat)) throw new ArgumentException("options.IndexFormat"); - if (string.IsNullOrWhiteSpace(options.TypeName)) throw new ArgumentException("options.TypeName"); if (string.IsNullOrWhiteSpace(options.TemplateName)) throw new ArgumentException("options.TemplateName"); + // Strip type argument if ESv7 since multiple types are not supported anymore + if (options.AutoRegisterTemplateVersion == AutoRegisterTemplateVersion.ESv7) + { + options.TypeName = "_doc"; + } + else + { + if (string.IsNullOrWhiteSpace(options.TypeName)) throw new ArgumentException("options.TypeName"); + } + _templateName = options.TemplateName; _templateMatchString = IndexFormatRegex.Replace(options.IndexFormat, @"$1*$2"); @@ -70,6 +79,7 @@ private ElasticsearchSinkState(ElasticsearchSinkOptions options) _options = options; + var configuration = new ConnectionConfiguration(options.ConnectionPool, options.Connection, options.Serializer) .RequestTimeout(options.ConnectionTimeout); @@ -112,7 +122,7 @@ public static ITextFormatter CreateDefaultDurableFormatter(ElasticsearchSinkOpti public string Serialize(object o) { - return _client.Serializer.SerializeToString(o, SerializationFormatting.None); + return _client.Serializer.SerializeToString(o, formatting: SerializationFormatting.None); } public string GetIndexForEvent(LogEvent e, DateTimeOffset offset) @@ -149,6 +159,7 @@ public void RegisterTemplateIfNeeded() } } + Console.WriteLine(_client.Serializer.SerializeToString(GetTemplateData())); var result = _client.IndicesPutTemplateForAll(_templateName, GetTempatePostData()); if (!result.Success) diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/SerializerAdapter.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/SerializerAdapter.cs index c551df02..c171a79b 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/SerializerAdapter.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/SerializerAdapter.cs @@ -53,7 +53,7 @@ public Task SerializeAsync(T data, Stream stream, public string SerializeToString(object value) { - return _elasticsearchSerializer.SerializeToString(value, SerializationFormatting.None); + return _elasticsearchSerializer.SerializeToString(value, formatting: SerializationFormatting.None); } } } \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj b/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj index 03f8600f..ed9333dd 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj @@ -24,9 +24,11 @@ + + @@ -39,7 +41,6 @@ - @@ -61,8 +62,19 @@ + + + + + + + + + + + $(DefineConstants);DOTNETCORE;NO_SERIALIZATION diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v7.json b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v7.json new file mode 100644 index 00000000..585f5323 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v7.json @@ -0,0 +1,75 @@ +{ + "index_patterns": [ "logstash-*" ], + "settings": { + "index.refresh_interval": "5s" + }, + "mappings": { + "dynamic_templates": [ + { + "numerics_in_fields": { + "path_match": "fields\\.[\\d+]$", + "match_pattern": "regex", + "mapping": { + "type": "text", + "index": true, + "norms": false + } + } + }, + { + "string_fields": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "text", + "index": true, + "norms": false, + "fields": { + "raw": { + "type": "keyword", + "index": true, + "ignore_above": 256 + } + } + } + } + } + ], + "properties": { + "message": { + "type": "text", + "index": "true" + }, + "exceptions": { + "type": "nested", + "properties": { + "Depth": { + "type": "integer" + }, + "RemoteStackIndex": { + "type": "integer" + }, + "HResult": { + "type": "integer" + }, + "StackTraceString": { + "type": "text", + "index": "true" + }, + "RemoteStackTraceString": { + "type": "text", + "index": "true" + }, + "ExceptionMessage": { + "type": "object", + "properties": { + "MemberType": { + "type": "integer" + } + } + } + } + } + } + } +} From f62bc73ec3fe2215aab3a35d82bfda84d3df2277 Mon Sep 17 00:00:00 2001 From: Daniel Rusznyak Date: Wed, 29 May 2019 17:12:17 +0100 Subject: [PATCH 02/15] Template V7 test --- .../Templating/Sendsv7TemplateTests.cs | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv7TemplateTests.cs diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv7TemplateTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv7TemplateTests.cs new file mode 100644 index 00000000..6180ed44 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv7TemplateTests.cs @@ -0,0 +1,66 @@ +using System; +using System.Reflection; +using FluentAssertions; +using Newtonsoft.Json.Linq; +using Xunit; + +namespace Serilog.Sinks.Elasticsearch.Tests.Templating +{ + public class Sendsv7TemplateTests : ElasticsearchSinkTestsBase + { + private readonly Tuple _templatePut; + + public Sendsv7TemplateTests() + { + _options.AutoRegisterTemplate = true; + _options.AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7; + + var loggerConfig = new LoggerConfiguration() + .MinimumLevel.Debug() + .Enrich.WithMachineName() + .WriteTo.ColoredConsole() + .WriteTo.Elasticsearch(_options); + + var logger = loggerConfig.CreateLogger(); + using (logger as IDisposable) + { + logger.Error("Test exception. Should not contain an embedded exception object."); + } + + this._seenHttpPosts.Should().NotBeNullOrEmpty().And.HaveCount(1); + this._seenHttpPuts.Should().NotBeNullOrEmpty().And.HaveCount(1); + _templatePut = this._seenHttpPuts[0]; + } + + [Fact] + public void ShouldRegisterTheCorrectTemplateOnRegistration() + { + + var method = typeof(Sendsv7TemplateTests).GetMethod(nameof(ShouldRegisterTheCorrectTemplateOnRegistration)); + JsonEquals(_templatePut.Item2, method, "template_v7.json"); + } + + [Fact] + public void TemplatePutToCorrectUrl() + { + var uri = _templatePut.Item1; + uri.AbsolutePath.Should().Be("/_template/serilog-events-template"); + } + + protected void JsonEquals(string json, MethodBase method, string fileName = null) + { +#if DOTNETCORE + var assembly = typeof(Sendsv7TemplateTests).GetTypeInfo().Assembly; +#else + var assembly = Assembly.GetExecutingAssembly(); +#endif + var expected = TestDataHelper.ReadEmbeddedResource(assembly, fileName ?? "template.json"); + + var nJson = JObject.Parse(json); + var nOtherJson = JObject.Parse(expected); + var equals = JToken.DeepEquals(nJson, nOtherJson); + if (equals) return; + expected.Should().BeEquivalentTo(json); + } + } +} \ No newline at end of file From ebd06c46a8fa2f21cadb391fb38d85515d27e391 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Thu, 4 Jul 2019 13:54:56 +0200 Subject: [PATCH 03/15] DurableElasticsearchsink existed twice using different casing --- .../ElasticSearch/DurableElasticSearchSink.cs | 98 ------------------- 1 file changed, 98 deletions(-) delete mode 100644 src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/DurableElasticSearchSink.cs diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/DurableElasticSearchSink.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/DurableElasticSearchSink.cs deleted file mode 100644 index 9ac18737..00000000 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/DurableElasticSearchSink.cs +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2014 Serilog Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Collections.Generic; -using System.Text; -using Elasticsearch.Net; -using Serilog.Core; -using Serilog.Events; - - -namespace Serilog.Sinks.Elasticsearch.Durable -{ - class DurableElasticsearchSink : ILogEventSink, IDisposable - { - // we rely on the date in the filename later! - const string FileNameSuffix = "-.json"; - - readonly Logger _sink; - readonly LogShipper> _shipper; - readonly ElasticsearchSinkState _state; - - public DurableElasticsearchSink(ElasticsearchSinkOptions options) - { - _state = ElasticsearchSinkState.Create(options); - - if (string.IsNullOrWhiteSpace(options.BufferBaseFilename)) - { - throw new ArgumentException("Cannot create the durable ElasticSearch sink without a buffer base file name!"); - } - - - _sink = new LoggerConfiguration() - .MinimumLevel.Verbose() - .WriteTo.File(_state.DurableFormatter, - options.BufferBaseFilename + FileNameSuffix, - rollingInterval: RollingInterval.Day, - fileSizeLimitBytes: options.BufferFileSizeLimitBytes, - rollOnFileSizeLimit: true, - retainedFileCountLimit: options.BufferFileCountLimit, - levelSwitch: _state.Options.LevelSwitch, - encoding: Encoding.UTF8) - .CreateLogger(); - - - var elasticSearchLogClient = new ElasticsearchLogClient( - elasticLowLevelClient: _state.Client, - cleanPayload: _state.Options.BufferCleanPayload); - - var payloadReader = new ElasticsearchPayloadReader( - pipelineName: _state.Options.PipelineName, - typeName:_state.Options.TypeName, - serialize:_state.Serialize, - getIndexForEvent: _state.GetBufferedIndexForEvent - ); - - _shipper = new ElasticsearchLogShipper( - bufferBaseFilename: _state.Options.BufferBaseFilename, - batchPostingLimit: _state.Options.BatchPostingLimit, - period: _state.Options.BufferLogShippingInterval ?? TimeSpan.FromSeconds(5), - eventBodyLimitBytes: _state.Options.SingleEventSizePostingLimit, - levelControlSwitch: _state.Options.LevelSwitch, - logClient: elasticSearchLogClient, - payloadReader: payloadReader, - retainedInvalidPayloadsLimitBytes: _state.Options.BufferRetainedInvalidPayloadsLimitBytes, - bufferSizeLimitBytes: _state.Options.BufferFileSizeLimitBytes, - registerTemplateIfNeeded: _state.RegisterTemplateIfNeeded); - - } - - public void Emit(LogEvent logEvent) - { - // This is a lagging indicator, but the network bandwidth usage benefits - // are worth the ambiguity. - if (_shipper.IsIncluded(logEvent)) - { - _sink.Write(logEvent); - } - } - - public void Dispose() - { - _sink.Dispose(); - _shipper.Dispose(); - } - } -} \ No newline at end of file From 81ad8899e81962cb08bd7d8ea945375a64175551 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Thu, 4 Jul 2019 23:23:42 +0200 Subject: [PATCH 04/15] Started on new integration tests project --- .editorconfig | 5 +- .../Bootstrap/ClientTestClusterBase.cs | 41 ++++++ .../Bootstrap/EphemeralClusterExtensions.cs | 54 ++++++++ ...SerilogSinkElasticsearchXunitRunOptions.cs | 107 +++++++++++++++ .../Bootstrap/XunitBootstrap.cs | 6 + .../Clusters/Elasticsearch7xCluster.cs | 23 ++++ .../Elasticsearch7X.cs | 79 +++++++++++ ...inks.Elasticsearch.IntegrationTests.csproj | 26 ++++ serilog-sinks-elasticsearch.sln | 6 + serilog-sinks-elasticsearch.sln.DotSettings | 4 + ...Serilog.Sinks.ElasticSearch.Symbols.nuspec | 2 +- .../Serilog.Sinks.ElasticSearch.nuspec | 2 +- .../Serilog.Sinks.Elasticsearch.csproj | 41 ++---- .../ElasticSearch/ElasticSearchLogClient.cs | 125 ------------------ .../ElasticSearch/ElasticSearchLogShipper.cs | 71 ---------- .../ElasticSearchPayloadReader.cs | 100 -------------- .../ElasticSearchTemplateProvider.cs | 12 +- .../ElasticSearch/ElasticsearchSinkOptions.cs | 15 ++- .../ElasticSearch/ElasticsearchSinkState.cs | 8 +- .../Serilog.Sinks.Elasticsearch.Tests.csproj | 23 +--- 20 files changed, 391 insertions(+), 359 deletions(-) create mode 100644 Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ClientTestClusterBase.cs create mode 100644 Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/EphemeralClusterExtensions.cs create mode 100644 Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/SerilogSinkElasticsearchXunitRunOptions.cs create mode 100644 Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/XunitBootstrap.cs create mode 100644 Serilog.Sinks.Elasticsearch.IntegrationTests/Clusters/Elasticsearch7xCluster.cs create mode 100644 Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7X.cs create mode 100644 Serilog.Sinks.Elasticsearch.IntegrationTests/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj create mode 100644 serilog-sinks-elasticsearch.sln.DotSettings delete mode 100644 src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchLogClient.cs delete mode 100644 src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchLogShipper.cs delete mode 100644 src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchPayloadReader.cs diff --git a/.editorconfig b/.editorconfig index 28e37c6f..9bdbb34e 100644 --- a/.editorconfig +++ b/.editorconfig @@ -2,4 +2,7 @@ root=true [*] indent_style = space -indent_size = 4 \ No newline at end of file +indent_size = 4 + +[*.csproj] +indent_size = 2 diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ClientTestClusterBase.cs b/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ClientTestClusterBase.cs new file mode 100644 index 00000000..c355ffbb --- /dev/null +++ b/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ClientTestClusterBase.cs @@ -0,0 +1,41 @@ +using Elastic.Managed.Ephemeral; +using Elastic.Managed.Ephemeral.Plugins; +using Elastic.Xunit; +using Nest; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap +{ + public abstract class ClientTestClusterBase : XunitClusterBase + { + protected ClientTestClusterBase(ClientTestClusterConfiguration configuration) : base(configuration) { } + + public IElasticClient Client => this.GetOrAddClient(s => ConnectionSettings(s)); + + protected virtual ConnectionSettings ConnectionSettings(ConnectionSettings s) => s; + } + + public class ClientTestClusterConfiguration : XunitClusterConfiguration + { + public ClientTestClusterConfiguration( + string elasticsearchVersion, + ClusterFeatures features = ClusterFeatures.None, + int numberOfNodes = 1, + params ElasticsearchPlugin[] plugins + ) + : base(elasticsearchVersion, features, new ElasticsearchPlugins(plugins), numberOfNodes) + { + HttpFiddlerAware = true; + CacheEsHomeInstallation = true; + + Add(AttributeKey("testingcluster"), "true"); + + Add($"script.max_compilations_per_minute", "10000", "<6.0.0-rc1"); + Add($"script.max_compilations_rate", "10000/1m", ">=6.0.0-rc1"); + + Add($"script.inline", "true", "<5.5.0"); + Add($"script.stored", "true", ">5.0.0-alpha1 <5.5.0"); + Add($"script.indexed", "true", "<5.0.0-alpha1"); + Add($"script.allowed_types", "inline,stored", ">=5.5.0"); + } + } +} diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/EphemeralClusterExtensions.cs b/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/EphemeralClusterExtensions.cs new file mode 100644 index 00000000..6db19ae3 --- /dev/null +++ b/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/EphemeralClusterExtensions.cs @@ -0,0 +1,54 @@ +using System; +using System.Diagnostics; +using System.Linq; +using System.Security.Cryptography.X509Certificates; +using Elastic.Managed.Ephemeral; +using Elastic.Xunit; +using Elasticsearch.Net; +using Nest; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap +{ + public static class EphemeralClusterExtensions + { + private static readonly bool RunningMitmProxy = Process.GetProcessesByName("mitmproxy").Any(); + private static readonly bool RunningFiddler = Process.GetProcessesByName("fiddler").Any(); + private static string LocalOrProxyHost => RunningFiddler || RunningMitmProxy ? "ipv4.fiddler" : "localhost"; + + public static ConnectionSettings CreateConnectionSettings(this IEphemeralCluster cluster) + where TConfig : EphemeralClusterConfiguration + { + var clusterNodes = cluster.NodesUris(LocalOrProxyHost); + return new ConnectionSettings(new StaticConnectionPool(clusterNodes)); + } + + public static IElasticClient GetOrAddClient( + this IEphemeralCluster cluster, + Func modifySettings = null + ) + where TConfig : EphemeralClusterConfiguration + { + modifySettings = modifySettings ?? (s => s); + return cluster.GetOrAddClient(c => + { + var settings = modifySettings(cluster.CreateConnectionSettings()); + + var current = (IConnectionConfigurationValues)settings; + var notAlreadyAuthenticated = current.BasicAuthenticationCredentials == null && current.ClientCertificates == null; + var noCertValidation = current.ServerCertificateValidationCallback == null; + + if (cluster.ClusterConfiguration.EnableSecurity && notAlreadyAuthenticated) + settings = settings.BasicAuthentication(ClusterAuthentication.Admin.Username, ClusterAuthentication.Admin.Password); + if (cluster.ClusterConfiguration.EnableSsl && noCertValidation) + { + //todo use CA callback instead of allow all + // ReSharper disable once UnusedVariable + var ca = new X509Certificate2(cluster.ClusterConfiguration.FileSystem.CaCertificate); + settings = settings.ServerCertificateValidationCallback(CertificateValidations.AllowAll); + } + var client = new ElasticClient(settings); + return client; + }); + } + } +} \ No newline at end of file diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/SerilogSinkElasticsearchXunitRunOptions.cs b/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/SerilogSinkElasticsearchXunitRunOptions.cs new file mode 100644 index 00000000..ea48e4c4 --- /dev/null +++ b/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/SerilogSinkElasticsearchXunitRunOptions.cs @@ -0,0 +1,107 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using Elastic.Xunit; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap +{ + /// Feeding TestClient.Configuration options to the runner + public class SerilogSinkElasticsearchXunitRunOptions : ElasticXunitRunOptions + { + public SerilogSinkElasticsearchXunitRunOptions() + { + RunIntegrationTests = true; + RunUnitTests = false; + ClusterFilter = null; + TestFilter = null; + IntegrationTestsMayUseAlreadyRunningNode = false; + } + + public override void OnBeforeTestsRun() { } + + public override void OnTestsFinished(Dictionary clusterTotals, ConcurrentBag> failedCollections) + { + Console.Out.Flush(); + DumpClusterTotals(clusterTotals); + DumpFailedCollections(failedCollections); + } + + private static void DumpClusterTotals(Dictionary clusterTotals) + { + Console.WriteLine("--------"); + Console.WriteLine("Individual cluster running times:"); + foreach (var kv in clusterTotals) Console.WriteLine($"- {kv.Key}: {kv.Value.Elapsed}"); + Console.WriteLine("--------"); + } + + private static void DumpFailedCollections(ConcurrentBag> failedCollections) + { + if (failedCollections.Count <= 0) return; + + Console.ForegroundColor = ConsoleColor.Red; + Console.WriteLine("Failed collections:"); + foreach (var t in failedCollections.OrderBy(p => p.Item1).ThenBy(t => t.Item2)) + + { + var cluster = t.Item1; + Console.WriteLine($" - {cluster}: {t.Item2}"); + } + DumpReproduceFilters(failedCollections); + Console.ResetColor(); + } + + private static void DumpReproduceFilters(ConcurrentBag> failedCollections) + { + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("---Reproduce: -----"); + var reproduceLine = ReproduceCommandLine(failedCollections); + Console.WriteLine(reproduceLine); + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TEAMCITY_VERSION"))) + Console.WriteLine($"##teamcity[buildProblem description='{reproduceLine}']"); + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TF_BUILD"))) + { + var count = failedCollections.Count; + Console.WriteLine($"##vso[task.logissue type=error;]{count} test failures"); + Console.WriteLine($"##vso[task.logissue type=error;]{reproduceLine}"); + } + Console.WriteLine("--------"); + } + + private static string ReproduceCommandLine(ConcurrentBag> failedCollections) + { + var sb = new StringBuilder("build.bat "); + + if (failedCollections.Count > 0) + { + var clusters = string.Join(",", failedCollections + .Select(c => c.Item1.ToLowerInvariant()) + .Distinct()); + sb.Append(" \""); + sb.Append(clusters); + sb.Append("\""); + } + + if ((failedCollections.Count < 30) && failedCollections.Count > 0) + { + sb.Append(" \""); + var tests = string.Join(",", failedCollections + .OrderBy(t => t.Item2) + .Select(c => c.Item2.ToLowerInvariant() + .Split('.') + .Last() + .Replace("apitests", "") + .Replace("usagetests", "") + .Replace("tests", "") + )); + sb.Append(tests); + sb.Append("\""); + } + + var reproduceLine = sb.ToString(); + return reproduceLine; + } + } +} diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/XunitBootstrap.cs b/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/XunitBootstrap.cs new file mode 100644 index 00000000..65de3e39 --- /dev/null +++ b/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/XunitBootstrap.cs @@ -0,0 +1,6 @@ +using Elastic.Xunit; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap; +using Xunit; + +[assembly: TestFramework("Elastic.Xunit.Sdk.ElasticTestFramework", "Elastic.Xunit")] +[assembly: ElasticXunitConfiguration(typeof(SerilogSinkElasticsearchXunitRunOptions))] diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Clusters/Elasticsearch7xCluster.cs b/Serilog.Sinks.Elasticsearch.IntegrationTests/Clusters/Elasticsearch7xCluster.cs new file mode 100644 index 00000000..8961e016 --- /dev/null +++ b/Serilog.Sinks.Elasticsearch.IntegrationTests/Clusters/Elasticsearch7xCluster.cs @@ -0,0 +1,23 @@ +using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Clusters +{ + /// + /// Use this cluster for APIs that do writes. If they are however intrusive or long running consider IntrusiveOperationCluster + /// instead. + /// + public class Elasticsearch7XCluster : ClientTestClusterBase + { + public Elasticsearch7XCluster() : base(CreateConfiguration()) { } + + private static ClientTestClusterConfiguration CreateConfiguration() + { + return new ClientTestClusterConfiguration("7.0.0") + { + MaxConcurrency = 1 + }; + } + + protected override void SeedCluster() { } + } +} diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7X.cs b/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7X.cs new file mode 100644 index 00000000..dd9d3249 --- /dev/null +++ b/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7X.cs @@ -0,0 +1,79 @@ +using System.Linq; +using Elastic.Managed.Ephemeral; +using Elastic.Xunit; +using Elastic.Xunit.XunitPlumbing; +using FluentAssertions; +using Nest; +using Serilog.Core; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Clusters; +using Xunit; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests +{ + public class Elasticsearch7XSetup + { + public const string IndexPrefix = "logs-7x-"; + public const string TemplateName = "serilog-logs-7x"; + public Elasticsearch7XSetup() + { + var loggerConfig = new LoggerConfiguration() + .MinimumLevel.Information() + .WriteTo.ColoredConsole() + .WriteTo.Elasticsearch(new ElasticsearchSinkOptions + { + + IndexFormat = IndexPrefix + "{0:yyyy.MM.dd}", + TemplateName = TemplateName, + AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7, + AutoRegisterTemplate = true + }); + var logger = loggerConfig.CreateLogger(); + + logger.Information("Hello Information"); + logger.Debug("Hello Debug"); + logger.Warning("Hello Warning"); + logger.Error("Hello Error"); + logger.Fatal("Hello Fatal"); + + logger.Dispose(); + } + + } + + + public class Elasticsearch7X : IClusterFixture, IClassFixture + { + private readonly Elasticsearch7XCluster _cluster; + private IElasticClient _client; + + public Elasticsearch7X(Elasticsearch7XCluster cluster, Elasticsearch7XSetup setup) + { + _cluster = cluster; + _client = cluster.Client; + } + + + [I] public void AssertTemplate() + { + var templateResponse = _client.Indices.GetTemplate(Elasticsearch7XSetup.TemplateName); + templateResponse.TemplateMappings.Should().NotBeEmpty(); + templateResponse.TemplateMappings.Keys.Should().Contain(Elasticsearch7XSetup.TemplateName); + + var template = templateResponse.TemplateMappings[Elasticsearch7XSetup.TemplateName]; + + template.IndexPatterns.Should().Contain(pattern => pattern.StartsWith(Elasticsearch7XSetup.IndexPrefix)); + + } + [I] public void AssertLogs() + { + var refreshed = _client.Indices.Refresh(Elasticsearch7XSetup.IndexPrefix + "*"); + + var search = _client.Search(s => s.Index(Elasticsearch7XSetup.IndexPrefix + "*")); + + // Informational should be filtered + search.Documents.Count().Should().Be(4); + + + } + } +} \ No newline at end of file diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj b/Serilog.Sinks.Elasticsearch.IntegrationTests/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj new file mode 100644 index 00000000..a6a57003 --- /dev/null +++ b/Serilog.Sinks.Elasticsearch.IntegrationTests/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj @@ -0,0 +1,26 @@ + + + + netcoreapp2.1;net461 + $(NoWarn);xUnit1013 + True + latest + True + + + + + + + + + + + + + + + + + + diff --git a/serilog-sinks-elasticsearch.sln b/serilog-sinks-elasticsearch.sln index 1f08366a..bb9350f1 100644 --- a/serilog-sinks-elasticsearch.sln +++ b/serilog-sinks-elasticsearch.sln @@ -24,6 +24,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serilog.Sinks.Elasticsearch EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serilog.Formatting.Elasticsearch", "src\Serilog.Formatting.Elasticsearch\Serilog.Formatting.Elasticsearch.csproj", "{0E6D34BF-322A-4803-94D1-355F6D5024BE}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serilog.Sinks.Elasticsearch.IntegrationTests", "Serilog.Sinks.Elasticsearch.IntegrationTests\Serilog.Sinks.Elasticsearch.IntegrationTests.csproj", "{23BC3821-E028-48B4-8F2C-83BB1B8B5525}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -46,6 +48,10 @@ Global {0E6D34BF-322A-4803-94D1-355F6D5024BE}.Debug|Any CPU.Build.0 = Debug|Any CPU {0E6D34BF-322A-4803-94D1-355F6D5024BE}.Release|Any CPU.ActiveCfg = Release|Any CPU {0E6D34BF-322A-4803-94D1-355F6D5024BE}.Release|Any CPU.Build.0 = Release|Any CPU + {23BC3821-E028-48B4-8F2C-83BB1B8B5525}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {23BC3821-E028-48B4-8F2C-83BB1B8B5525}.Debug|Any CPU.Build.0 = Debug|Any CPU + {23BC3821-E028-48B4-8F2C-83BB1B8B5525}.Release|Any CPU.ActiveCfg = Release|Any CPU + {23BC3821-E028-48B4-8F2C-83BB1B8B5525}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/serilog-sinks-elasticsearch.sln.DotSettings b/serilog-sinks-elasticsearch.sln.DotSettings new file mode 100644 index 00000000..95859094 --- /dev/null +++ b/serilog-sinks-elasticsearch.sln.DotSettings @@ -0,0 +1,4 @@ + + True + True + True \ No newline at end of file diff --git a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.Symbols.nuspec b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.Symbols.nuspec index 289c659a..9932b8fb 100644 --- a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.Symbols.nuspec +++ b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.Symbols.nuspec @@ -12,7 +12,7 @@ serilog logging elasticsearch - + diff --git a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.nuspec b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.nuspec index 5c6c8148..c6ba9cc7 100644 --- a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.nuspec +++ b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.nuspec @@ -13,7 +13,7 @@ serilog logging elasticsearch - + diff --git a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj index d2e3559f..5826a20f 100644 --- a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj +++ b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj @@ -4,7 +4,7 @@ 6.0.0 alpha Michiel van Oudheusden, Martijn Laarman, Mogens Heller Grabe, Serilog Contributors - net45;netstandard1.3;netstandard2.0 + net461;netstandard2.0 true true Serilog.Sinks.Elasticsearch @@ -28,31 +28,6 @@ false - - - - - - - - - - - - - - - - - - - - - - - - - 1591;1701;1702 $(DefineConstants);DURABLE;THREADING_TIMER @@ -63,7 +38,7 @@ $(DefineConstants);DURABLE;THREADING_TIMER - + 1591;1701;1702 $(DefineConstants);DURABLE;THREADING_TIMER;HRESULTS @@ -75,10 +50,18 @@ - - + + + + + + + + + + diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchLogClient.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchLogClient.cs deleted file mode 100644 index 6b2d45bf..00000000 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchLogClient.cs +++ /dev/null @@ -1,125 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.ExceptionServices; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using Elasticsearch.Net; -using Serilog.Debugging; - -namespace Serilog.Sinks.Elasticsearch.Durable -{ - /// - /// - /// - public class ElasticsearchLogClient : ILogClient> - { - private readonly IElasticLowLevelClient _elasticLowLevelClient; - private readonly Func _cleanPayload; - - /// - /// - /// - /// - /// - public ElasticsearchLogClient(IElasticLowLevelClient elasticLowLevelClient, - Func cleanPayload) - { - _elasticLowLevelClient = elasticLowLevelClient; - _cleanPayload = cleanPayload; - } - - public async Task SendPayloadAsync(List payload) - { - return await SendPayloadAsync(payload, true); - } - - public async Task SendPayloadAsync(List payload,bool first) - { - try - { - if (payload == null || !payload.Any()) return new SentPayloadResult(null, true); - var response = await _elasticLowLevelClient.BulkAsync(PostData.MultiJson(payload)); - - if (response.Success) - { - var cleanPayload = new List(); - var invalidPayload = GetInvalidPayloadAsync(response, payload,out cleanPayload); - if ((cleanPayload?.Any() ?? false) && first) - { - await SendPayloadAsync(cleanPayload,false); - } - - return new SentPayloadResult(response, true, invalidPayload); - } - else - { - SelfLog.WriteLine("Received failed ElasticSearch shipping result {0}: {1}", response.HttpStatusCode, - response.OriginalException); - return new SentPayloadResult(response, false, - new InvalidResult() - { - StatusCode = response.HttpStatusCode ?? 500, - Content = response.OriginalException.ToString() - }); - } - } - catch (Exception ex) - { - SelfLog.WriteLine("Exception while emitting periodic batch from {0}: {1}", this, ex); - return new SentPayloadResult(null, false, null, ex); - } - - - } - - private InvalidResult GetInvalidPayloadAsync(DynamicResponse baseResult, List payload, out List cleanPayload) - { - int i = 0; - cleanPayload = new List(); - var items = baseResult.Body["items"]; - if (items == null) return null; - List badPayload = new List(); - - bool hasErrors = false; - foreach (dynamic item in items) - { - long? status = item.index?.status; - i++; - if (!status.HasValue || status < 300) - { - continue; - } - - hasErrors = true; - var id = item.index?._id; - var error = item.index?.error; - if (int.TryParse(id.Split('_')[0], out int index)) - { - SelfLog.WriteLine("Received failed ElasticSearch shipping result {0}: {1}. Failed payload : {2}.", status, error?.ToString(), payload.ElementAt(index * 2 + 1)); - badPayload.Add(payload.ElementAt(index * 2)); - badPayload.Add(payload.ElementAt(index * 2 + 1)); - if (_cleanPayload != null) - { - cleanPayload.Add(payload.ElementAt(index * 2)); - cleanPayload.Add(_cleanPayload(payload.ElementAt(index * 2 + 1), status, error?.ToString())); - } - } - else - { - SelfLog.WriteLine($"Received failed ElasticSearch shipping result {status}: {error?.ToString()}."); - } - } - - if (!hasErrors) - return null; - return new InvalidResult() - { - StatusCode = baseResult.HttpStatusCode ?? 500, - Content = baseResult.ToString(), - BadPayLoad = String.Join(Environment.NewLine, badPayload) - }; - } - } -} diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchLogShipper.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchLogShipper.cs deleted file mode 100644 index 6a6b8b26..00000000 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchLogShipper.cs +++ /dev/null @@ -1,71 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Serilog.Core; -using Serilog.Debugging; -using Serilog.Sinks.Elasticsearch.Durable; - -namespace Serilog.Sinks.Elasticsearch.Durable -{ - /// - /// - /// - public class ElasticsearchLogShipper : LogShipper> - { - private readonly Action _registerTemplateIfNeeded; - bool _didRegisterTemplateIfNeeded = false; - - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - public ElasticsearchLogShipper(string bufferBaseFilename, int batchPostingLimit, TimeSpan period, - long? eventBodyLimitBytes, LoggingLevelSwitch levelControlSwitch, ILogClient> logClient, - IPayloadReader> payloadReader, long? retainedInvalidPayloadsLimitBytes, - long? bufferSizeLimitBytes, Action registerTemplateIfNeeded) - : base(bufferBaseFilename, batchPostingLimit, period, eventBodyLimitBytes, - levelControlSwitch, logClient, payloadReader, retainedInvalidPayloadsLimitBytes, bufferSizeLimitBytes) - { - _registerTemplateIfNeeded = registerTemplateIfNeeded; - } - - /// - /// - /// - /// - protected override async Task OnTick() - { - bool success = true; - try - { - if (!_didRegisterTemplateIfNeeded) - { - if (_registerTemplateIfNeeded != null) - { - _registerTemplateIfNeeded(); - _didRegisterTemplateIfNeeded = true; - } - } - } - catch (Exception ex) - { - SelfLog.WriteLine("Exception while emitting periodic batch from {0}: {1}", this, ex); - _connectionSchedule.MarkFailure(); - success = false; - } - if (success) - await base.OnTick(); - } - } -} diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchPayloadReader.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchPayloadReader.cs deleted file mode 100644 index a31b253e..00000000 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticSearchPayloadReader.cs +++ /dev/null @@ -1,100 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Serilog.Sinks.Elasticsearch.Durable -{ - /// - /// - /// - public class ElasticsearchPayloadReader: APayloadReader> - { - private readonly string _pipelineName; - private readonly string _typeName; - private readonly Func _serialize; - private readonly Func _getIndexForEvent; - private List _payload; - private int _count; - private DateTime _date; - - /// - /// - /// - /// - /// - /// - /// - public ElasticsearchPayloadReader(string pipelineName,string typeName, Func serialize,Func getIndexForEvent) - { - _pipelineName = pipelineName; - _typeName = typeName; - _serialize = serialize; - _getIndexForEvent = getIndexForEvent; - } - - /// - /// - /// - /// - public override List GetNoPayload() - { - return new List(); - } - - /// - /// - /// - /// - protected override void InitPayLoad(string filename) - { - _payload = new List(); - _count = 0; - var lastToken = filename.Split('-').Last(); - - // lastToken should be something like 20150218.json or 20150218_3.json now - if (!lastToken.ToLowerInvariant().EndsWith(".json")) - { - throw new FormatException(string.Format("The file name '{0}' does not seem to follow the right file pattern - it must be named [whatever]-{{Date}}[_n].json", Path.GetFileName(filename))); - } - - var dateString = lastToken.Substring(0, 8); - _date = DateTime.ParseExact(dateString, "yyyyMMdd", CultureInfo.InvariantCulture); - } - /// - /// - /// - /// - protected override List FinishPayLoad() - { - return _payload; - } - - /// - /// - /// - /// - protected override void AddToPayLoad(string nextLine) - { - var indexName = _getIndexForEvent(nextLine, _date); - var action = default(object); - - if (string.IsNullOrWhiteSpace(_pipelineName)) - { - action = new { index = new { _index = indexName, _type = _typeName, _id = _count + "_" + Guid.NewGuid() } }; - } - else - { - action = new { index = new { _index = indexName, _type = _typeName, _id = _count + "_" + Guid.NewGuid(), pipeline = _pipelineName } }; - } - - var actionJson = _serialize(action); - _payload.Add(actionJson); - _payload.Add(nextLine); - _count++; - } - } -} diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs index 0325992b..e247e8b7 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs @@ -118,7 +118,7 @@ private static object GetTemplateESv7( }, properties = new Dictionary { - {"message", new {type = "text", index = "true"}}, + {"message", new {type = "text", index = true}}, { "exceptions", new { @@ -128,8 +128,8 @@ private static object GetTemplateESv7( {"Depth", new {type = "integer"}}, {"RemoteStackIndex", new {type = "integer"}}, {"HResult", new {type = "integer"}}, - {"StackTraceString", new {type = "text", index = "true"}}, - {"RemoteStackTraceString", new {type = "text", index = "true"}}, + {"StackTraceString", new {type = "text", index = true}}, + {"RemoteStackTraceString", new {type = "text", index = true}}, { "ExceptionMessage", new { @@ -210,7 +210,7 @@ private static object GetTemplateESv6( }, properties = new Dictionary { - {"message", new {type = "text", index = "true"}}, + {"message", new {type = "text", index = true}}, { "exceptions", new { @@ -220,8 +220,8 @@ private static object GetTemplateESv6( {"Depth", new {type = "integer"}}, {"RemoteStackIndex", new {type = "integer"}}, {"HResult", new {type = "integer"}}, - {"StackTraceString", new {type = "text", index = "true"}}, - {"RemoteStackTraceString", new {type = "text", index = "true"}}, + {"StackTraceString", new {type = "text", index = true}}, + {"RemoteStackTraceString", new {type = "text", index = true}}, { "ExceptionMessage", new { diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs index 8397d403..29ccb503 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs @@ -271,6 +271,7 @@ public ElasticsearchSinkOptions() this.BufferFileCountLimit = 31; this.BufferFileSizeLimitBytes = 100L * 1024 * 1024; this.FormatStackTraceAsArray = false; + this.ConnectionPool = new SingleNodeConnectionPool(_defaultNode); } /// @@ -290,13 +291,13 @@ public ElasticsearchSinkOptions(IConnectionPool connectionPool) public ElasticsearchSinkOptions(IEnumerable nodes) : this() { - nodes = nodes != null && nodes.Any(n => n != null) - ? nodes.Where(n => n != null) - : new[] { new Uri("http://localhost:9200") }; - if (nodes.Count() == 1) - ConnectionPool = new SingleNodeConnectionPool(nodes.First()); + var materialized = nodes?.Where(n => n != null).ToArray(); + if (materialized == null || materialized.Length == 0) + materialized = new[] { _defaultNode }; + if (materialized.Length == 1) + ConnectionPool = new SingleNodeConnectionPool(materialized.First()); else - ConnectionPool = new StaticConnectionPool(nodes); + ConnectionPool = new StaticConnectionPool(materialized); } /// @@ -304,6 +305,8 @@ public ElasticsearchSinkOptions(IEnumerable nodes) /// /// The node to write to public ElasticsearchSinkOptions(Uri node) : this(new[] { node }) { } + + private readonly Uri _defaultNode = new Uri("http://localhost:9200"); } /// diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs index 679857c9..1b0249e9 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs @@ -17,6 +17,7 @@ using System.Text; using System.Text.RegularExpressions; using Elasticsearch.Net; +using Elasticsearch.Net.Specification.IndicesApi; using Serilog.Debugging; using Serilog.Events; using Serilog.Formatting; @@ -150,7 +151,10 @@ public void RegisterTemplateIfNeeded() { if (!_options.OverwriteTemplate) { - var templateExistsResponse = _client.IndicesExistsTemplateForAll(_templateName); + var templateExistsResponse = _client.Indices.TemplateExistsForAll(_templateName, new IndexTemplateExistsRequestParameters() + { + RequestConfiguration = new RequestConfiguration() { AllowedStatusCodes = new [] {200, 404} } + }); if (templateExistsResponse.HttpStatusCode == 200) { TemplateRegistrationSuccess = true; @@ -160,7 +164,7 @@ public void RegisterTemplateIfNeeded() } Console.WriteLine(_client.Serializer.SerializeToString(GetTemplateData())); - var result = _client.IndicesPutTemplateForAll(_templateName, GetTempatePostData()); + var result = _client.Indices.PutTemplateForAll(_templateName, GetTempatePostData()); if (!result.Success) { diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj b/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj index ed9333dd..a4c073c3 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj @@ -1,7 +1,7 @@  - netcoreapp1.1;netcoreapp2.0;net451 + netcoreapp2.0;net461 Serilog.Sinks.Elasticsearch.Tests Serilog.Sinks.Elasticsearch.Tests true @@ -41,14 +41,17 @@ + + - - + + + @@ -62,31 +65,17 @@ - - - - - - - - $(DefineConstants);DOTNETCORE;NO_SERIALIZATION - - $(DefineConstants);DOTNETCORE;PARTIALLY_SERIALIZATION - - - - From 6c2497e2febd19d43b1288a29cb12cdf0221ad93 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Mon, 8 Jul 2019 13:18:48 +0200 Subject: [PATCH 05/15] Integration tests for 6.x and refactored common bits --- .../Bootstrap/ClientTestClusterBase.cs | 4 - .../ElasticsearchSinkOptionsFactory.cs | 36 +++++++++ .../Bootstrap/EphemeralClusterExtensions.cs | 54 ------------- .../Bootstrap/ProxyDetection.cs | 15 ++++ .../Bootstrap/Elasticsearch6XCluster.cs | 19 +++++ .../Bootstrap/Elasticsearch6XTestBase.cs | 50 ++++++++++++ .../Elasticsearch6/Elasticsearch6X.cs | 71 +++++++++++++++++ .../Bootstrap/Elasticsearch7XCluster.cs} | 6 +- .../Bootstrap/Elasticsearch7XTestBase.cs | 51 ++++++++++++ .../Elasticsearch7/Elasticsearch7X.cs | 67 ++++++++++++++++ .../Elasticsearch7X.cs | 79 ------------------- ...inks.Elasticsearch.IntegrationTests.csproj | 9 ++- nuget.config | 2 + serilog-sinks-elasticsearch.sln.DotSettings | 4 + .../ElasticSearch/ElasticsearchSinkOptions.cs | 10 ++- 15 files changed, 331 insertions(+), 146 deletions(-) create mode 100644 Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ElasticsearchSinkOptionsFactory.cs delete mode 100644 Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/EphemeralClusterExtensions.cs create mode 100644 Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ProxyDetection.cs create mode 100644 Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Bootstrap/Elasticsearch6XCluster.cs create mode 100644 Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Bootstrap/Elasticsearch6XTestBase.cs create mode 100644 Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6X.cs rename Serilog.Sinks.Elasticsearch.IntegrationTests/{Clusters/Elasticsearch7xCluster.cs => Elasticsearch7/Bootstrap/Elasticsearch7XCluster.cs} (63%) create mode 100644 Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Bootstrap/Elasticsearch7XTestBase.cs create mode 100644 Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7X.cs delete mode 100644 Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7X.cs diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ClientTestClusterBase.cs b/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ClientTestClusterBase.cs index c355ffbb..313b6f3a 100644 --- a/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ClientTestClusterBase.cs +++ b/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ClientTestClusterBase.cs @@ -8,10 +8,6 @@ namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap public abstract class ClientTestClusterBase : XunitClusterBase { protected ClientTestClusterBase(ClientTestClusterConfiguration configuration) : base(configuration) { } - - public IElasticClient Client => this.GetOrAddClient(s => ConnectionSettings(s)); - - protected virtual ConnectionSettings ConnectionSettings(ConnectionSettings s) => s; } public class ClientTestClusterConfiguration : XunitClusterConfiguration diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ElasticsearchSinkOptionsFactory.cs b/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ElasticsearchSinkOptionsFactory.cs new file mode 100644 index 00000000..27279840 --- /dev/null +++ b/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ElasticsearchSinkOptionsFactory.cs @@ -0,0 +1,36 @@ +using System; +using Elasticsearch.Net; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap +{ + public static class ElasticsearchSinkOptionsFactory + { + public static ElasticsearchSinkOptions Create(string indexPrefix, string templateName, Action alterOptions = null) + { + // make sure we run through `ipv4.fiddler` if `fiddler` or `mitmproxy` is running + // NOTE with the latter you need to add `ipv4.fiddler` as an alias to 127.0.0.1 in your HOSTS file manually + var pool = new SingleNodeConnectionPool(new Uri($"http://{ProxyDetection.LocalOrProxyHost}:9200")); + var options = new ElasticsearchSinkOptions(pool) + { + IndexFormat = indexPrefix + "{0:yyyy.MM.dd}", + TemplateName = templateName, + }; + + alterOptions?.Invoke(options); + // here we make sure we set a proxy on the elasticsearch connection + // when we detect `mitmproxy` running. This is a cli tool similar to fiddler + // on *nix systems which aids debugging what goes over the wire + var provided = options.ModifyConnectionSettings; + options.ModifyConnectionSettings = configuration => + { + if (ProxyDetection.RunningMitmProxy) + configuration.Proxy(ProxyDetection.MitmProxyAddress, null, (string) null); + configuration = provided?.Invoke(configuration) ?? configuration; + return configuration; + }; + + return options; + } + + } +} \ No newline at end of file diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/EphemeralClusterExtensions.cs b/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/EphemeralClusterExtensions.cs deleted file mode 100644 index 6db19ae3..00000000 --- a/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/EphemeralClusterExtensions.cs +++ /dev/null @@ -1,54 +0,0 @@ -using System; -using System.Diagnostics; -using System.Linq; -using System.Security.Cryptography.X509Certificates; -using Elastic.Managed.Ephemeral; -using Elastic.Xunit; -using Elasticsearch.Net; -using Nest; - -namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap -{ - public static class EphemeralClusterExtensions - { - private static readonly bool RunningMitmProxy = Process.GetProcessesByName("mitmproxy").Any(); - private static readonly bool RunningFiddler = Process.GetProcessesByName("fiddler").Any(); - private static string LocalOrProxyHost => RunningFiddler || RunningMitmProxy ? "ipv4.fiddler" : "localhost"; - - public static ConnectionSettings CreateConnectionSettings(this IEphemeralCluster cluster) - where TConfig : EphemeralClusterConfiguration - { - var clusterNodes = cluster.NodesUris(LocalOrProxyHost); - return new ConnectionSettings(new StaticConnectionPool(clusterNodes)); - } - - public static IElasticClient GetOrAddClient( - this IEphemeralCluster cluster, - Func modifySettings = null - ) - where TConfig : EphemeralClusterConfiguration - { - modifySettings = modifySettings ?? (s => s); - return cluster.GetOrAddClient(c => - { - var settings = modifySettings(cluster.CreateConnectionSettings()); - - var current = (IConnectionConfigurationValues)settings; - var notAlreadyAuthenticated = current.BasicAuthenticationCredentials == null && current.ClientCertificates == null; - var noCertValidation = current.ServerCertificateValidationCallback == null; - - if (cluster.ClusterConfiguration.EnableSecurity && notAlreadyAuthenticated) - settings = settings.BasicAuthentication(ClusterAuthentication.Admin.Username, ClusterAuthentication.Admin.Password); - if (cluster.ClusterConfiguration.EnableSsl && noCertValidation) - { - //todo use CA callback instead of allow all - // ReSharper disable once UnusedVariable - var ca = new X509Certificate2(cluster.ClusterConfiguration.FileSystem.CaCertificate); - settings = settings.ServerCertificateValidationCallback(CertificateValidations.AllowAll); - } - var client = new ElasticClient(settings); - return client; - }); - } - } -} \ No newline at end of file diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ProxyDetection.cs b/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ProxyDetection.cs new file mode 100644 index 00000000..0ba27742 --- /dev/null +++ b/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ProxyDetection.cs @@ -0,0 +1,15 @@ +using System; +using System.Diagnostics; +using System.Linq; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap +{ + public static class ProxyDetection + { + public static readonly bool RunningMitmProxy = Process.GetProcessesByName("mitmproxy").Any(); + private static readonly bool RunningFiddler = Process.GetProcessesByName("fiddler").Any(); + public static string LocalOrProxyHost => RunningFiddler || RunningMitmProxy ? "ipv4.fiddler" : "localhost"; + + public static readonly Uri MitmProxyAddress = new Uri("http://localhost:8080"); + } +} \ No newline at end of file diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Bootstrap/Elasticsearch6XCluster.cs b/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Bootstrap/Elasticsearch6XCluster.cs new file mode 100644 index 00000000..30d6d229 --- /dev/null +++ b/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Bootstrap/Elasticsearch6XCluster.cs @@ -0,0 +1,19 @@ +using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch6.Bootstrap +{ + public class Elasticsearch6XCluster : ClientTestClusterBase + { + public Elasticsearch6XCluster() : base(CreateConfiguration()) { } + + private static ClientTestClusterConfiguration CreateConfiguration() + { + return new ClientTestClusterConfiguration("6.6.0") + { + MaxConcurrency = 1 + }; + } + + protected override void SeedCluster() { } + } +} diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Bootstrap/Elasticsearch6XTestBase.cs b/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Bootstrap/Elasticsearch6XTestBase.cs new file mode 100644 index 00000000..bc4f95d9 --- /dev/null +++ b/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Bootstrap/Elasticsearch6XTestBase.cs @@ -0,0 +1,50 @@ +using System; +using System.Security.Cryptography.X509Certificates; +using Elastic.Managed.Ephemeral; +using Elastic.Xunit; +using Elastic.Xunit.XunitPlumbing; +using Elasticsearch.Net6; +using Nest6; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch6.Bootstrap +{ + public abstract class Elasticsearch6XTestBase : IClusterFixture + { + protected Elasticsearch6XTestBase(Elasticsearch6XCluster cluster) => Cluster = cluster; + + private Elasticsearch6XCluster Cluster { get; } + + protected IElasticClient Client => this.CreateClient(); + + protected virtual ConnectionSettings SetClusterSettings(ConnectionSettings s) => s; + + private IElasticClient CreateClient() => + Cluster.GetOrAddClient(c => + { + var clusterNodes = c.NodesUris(ProxyDetection.LocalOrProxyHost); + var settings = new ConnectionSettings(new StaticConnectionPool(clusterNodes)); + if (ProxyDetection.RunningMitmProxy) + settings = settings.Proxy(new Uri("http://localhost:8080"), null, null); + settings = SetClusterSettings(settings); + + var current = (IConnectionConfigurationValues)settings; + var notAlreadyAuthenticated = current.BasicAuthenticationCredentials == null && current.ClientCertificates == null; + var noCertValidation = current.ServerCertificateValidationCallback == null; + + var config = Cluster.ClusterConfiguration; + if (config.EnableSecurity && notAlreadyAuthenticated) + settings = settings.BasicAuthentication(ClusterAuthentication.Admin.Username, ClusterAuthentication.Admin.Password); + if (config.EnableSsl && noCertValidation) + { + //todo use CA callback instead of allow all + // ReSharper disable once UnusedVariable + var ca = new X509Certificate2(config.FileSystem.CaCertificate); + settings = settings.ServerCertificateValidationCallback(CertificateValidations.AllowAll); + } + var client = new ElasticClient(settings); + return client; + }); + } + +} diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6X.cs b/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6X.cs new file mode 100644 index 00000000..0ac38608 --- /dev/null +++ b/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6X.cs @@ -0,0 +1,71 @@ +using System.Linq; +using Elastic.Xunit.XunitPlumbing; +using FluentAssertions; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch6.Bootstrap; +using Xunit; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch6 +{ + public class Elasticsearch6X : Elasticsearch6XTestBase, IClassFixture + { + private readonly SetupSerilog _setup; + + public Elasticsearch6X(Elasticsearch6XCluster cluster, SetupSerilog setup) : base(cluster) => _setup = setup; + + + [I] public void AssertTemplate() + { + var templateResponse = Client.GetIndexTemplate(t=>t.Name(SetupSerilog.TemplateName)); + templateResponse.TemplateMappings.Should().NotBeEmpty(); + templateResponse.TemplateMappings.Keys.Should().Contain(SetupSerilog.TemplateName); + + var template = templateResponse.TemplateMappings[SetupSerilog.TemplateName]; + + template.IndexPatterns.Should().Contain(pattern => pattern.StartsWith(SetupSerilog.IndexPrefix)); + } + + [I] public void AssertLogs() + { + var refreshed = Client.Refresh(SetupSerilog.IndexPrefix + "*"); + + var search = Client.Search(s => s + .Index(SetupSerilog.IndexPrefix + "*") + .Type(ElasticsearchSinkOptions.DefaultTypeName) + ); + + // Informational should be filtered + search.Documents.Count().Should().Be(4); + } + + // ReSharper disable once ClassNeverInstantiated.Global + public class SetupSerilog + { + public const string IndexPrefix = "logs-6x-"; + public const string TemplateName = "serilog-logs-6x"; + + public SetupSerilog() + { + var loggerConfig = new LoggerConfiguration() + .MinimumLevel.Information() + .WriteTo.ColoredConsole() + .WriteTo.Elasticsearch( + ElasticsearchSinkOptionsFactory.Create(IndexPrefix, TemplateName, o => + { + o.AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv6; + o.AutoRegisterTemplate = true; + }) + ); + var logger = loggerConfig.CreateLogger(); + + logger.Information("Hello Information"); + logger.Debug("Hello Debug"); + logger.Warning("Hello Warning"); + logger.Error("Hello Error"); + logger.Fatal("Hello Fatal"); + + logger.Dispose(); + } + } + } +} \ No newline at end of file diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Clusters/Elasticsearch7xCluster.cs b/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Bootstrap/Elasticsearch7XCluster.cs similarity index 63% rename from Serilog.Sinks.Elasticsearch.IntegrationTests/Clusters/Elasticsearch7xCluster.cs rename to Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Bootstrap/Elasticsearch7XCluster.cs index 8961e016..b1f2ab50 100644 --- a/Serilog.Sinks.Elasticsearch.IntegrationTests/Clusters/Elasticsearch7xCluster.cs +++ b/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Bootstrap/Elasticsearch7XCluster.cs @@ -1,11 +1,7 @@ using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap; -namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Clusters +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch7.Bootstrap { - /// - /// Use this cluster for APIs that do writes. If they are however intrusive or long running consider IntrusiveOperationCluster - /// instead. - /// public class Elasticsearch7XCluster : ClientTestClusterBase { public Elasticsearch7XCluster() : base(CreateConfiguration()) { } diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Bootstrap/Elasticsearch7XTestBase.cs b/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Bootstrap/Elasticsearch7XTestBase.cs new file mode 100644 index 00000000..281c88fd --- /dev/null +++ b/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Bootstrap/Elasticsearch7XTestBase.cs @@ -0,0 +1,51 @@ +using System; +using System.Security.Cryptography.X509Certificates; +using Elastic.Managed.Ephemeral; +using Elastic.Xunit; +using Elastic.Xunit.XunitPlumbing; +using Elasticsearch.Net; +using Nest; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap; +using Xunit; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch7.Bootstrap +{ + public abstract class Elasticsearch7XTestBase : IClusterFixture + { + protected Elasticsearch7XTestBase(Elasticsearch7XCluster cluster) => Cluster = cluster; + + private Elasticsearch7XCluster Cluster { get; } + + protected IElasticClient Client => this.CreateClient(); + + protected virtual ConnectionSettings SetClusterSettings(ConnectionSettings s) => s; + + private IElasticClient CreateClient() => + Cluster.GetOrAddClient(c => + { + var clusterNodes = c.NodesUris(ProxyDetection.LocalOrProxyHost); + var settings = new ConnectionSettings(new StaticConnectionPool(clusterNodes)); + if (ProxyDetection.RunningMitmProxy) + settings = settings.Proxy(new Uri("http://localhost:8080"), null, (string)null); + settings = SetClusterSettings(settings); + + var current = (IConnectionConfigurationValues)settings; + var notAlreadyAuthenticated = current.BasicAuthenticationCredentials == null && current.ClientCertificates == null; + var noCertValidation = current.ServerCertificateValidationCallback == null; + + var config = Cluster.ClusterConfiguration; + if (config.EnableSecurity && notAlreadyAuthenticated) + settings = settings.BasicAuthentication(ClusterAuthentication.Admin.Username, ClusterAuthentication.Admin.Password); + if (config.EnableSsl && noCertValidation) + { + //todo use CA callback instead of allow all + // ReSharper disable once UnusedVariable + var ca = new X509Certificate2(config.FileSystem.CaCertificate); + settings = settings.ServerCertificateValidationCallback(CertificateValidations.AllowAll); + } + var client = new ElasticClient(settings); + return client; + }); + } + +} diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7X.cs b/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7X.cs new file mode 100644 index 00000000..f025546d --- /dev/null +++ b/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7X.cs @@ -0,0 +1,67 @@ +using System.Linq; +using Elastic.Xunit.XunitPlumbing; +using FluentAssertions; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch7.Bootstrap; +using Xunit; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch7 +{ + public class Elasticsearch7X : Elasticsearch7XTestBase, IClassFixture + { + private readonly SetupSerilog _setup; + + public Elasticsearch7X(Elasticsearch7XCluster cluster, SetupSerilog setup) : base(cluster) => _setup = setup; + + [I] public void AssertTemplate() + { + var templateResponse = Client.Indices.GetTemplate(SetupSerilog.TemplateName); + templateResponse.TemplateMappings.Should().NotBeEmpty(); + templateResponse.TemplateMappings.Keys.Should().Contain(SetupSerilog.TemplateName); + + var template = templateResponse.TemplateMappings[SetupSerilog.TemplateName]; + + template.IndexPatterns.Should().Contain(pattern => pattern.StartsWith(SetupSerilog.IndexPrefix)); + } + + [I] public void AssertLogs() + { + var refreshed = Client.Indices.Refresh(SetupSerilog.IndexPrefix + "*"); + + var search = Client.Search(s => s.Index(SetupSerilog.IndexPrefix + "*")); + + // Informational should be filtered + search.Documents.Count().Should().Be(4); + } + + // ReSharper disable once ClassNeverInstantiated.Global + public class SetupSerilog + { + public const string IndexPrefix = "logs-7x-"; + public const string TemplateName = "serilog-logs-7x"; + + public SetupSerilog() + { + var loggerConfig = new LoggerConfiguration() + .MinimumLevel.Information() + .WriteTo.ColoredConsole() + .WriteTo.Elasticsearch( + ElasticsearchSinkOptionsFactory.Create(IndexPrefix, TemplateName, o => + { + o.AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7; + o.AutoRegisterTemplate = true; + }) + ); + using (var logger = loggerConfig.CreateLogger()) + { + logger.Information("Hello Information"); + logger.Debug("Hello Debug"); + logger.Warning("Hello Warning"); + logger.Error("Hello Error"); + logger.Fatal("Hello Fatal"); + } + } + } + + } +} \ No newline at end of file diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7X.cs b/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7X.cs deleted file mode 100644 index dd9d3249..00000000 --- a/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7X.cs +++ /dev/null @@ -1,79 +0,0 @@ -using System.Linq; -using Elastic.Managed.Ephemeral; -using Elastic.Xunit; -using Elastic.Xunit.XunitPlumbing; -using FluentAssertions; -using Nest; -using Serilog.Core; -using Serilog.Sinks.Elasticsearch.IntegrationTests.Clusters; -using Xunit; - -namespace Serilog.Sinks.Elasticsearch.IntegrationTests -{ - public class Elasticsearch7XSetup - { - public const string IndexPrefix = "logs-7x-"; - public const string TemplateName = "serilog-logs-7x"; - public Elasticsearch7XSetup() - { - var loggerConfig = new LoggerConfiguration() - .MinimumLevel.Information() - .WriteTo.ColoredConsole() - .WriteTo.Elasticsearch(new ElasticsearchSinkOptions - { - - IndexFormat = IndexPrefix + "{0:yyyy.MM.dd}", - TemplateName = TemplateName, - AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7, - AutoRegisterTemplate = true - }); - var logger = loggerConfig.CreateLogger(); - - logger.Information("Hello Information"); - logger.Debug("Hello Debug"); - logger.Warning("Hello Warning"); - logger.Error("Hello Error"); - logger.Fatal("Hello Fatal"); - - logger.Dispose(); - } - - } - - - public class Elasticsearch7X : IClusterFixture, IClassFixture - { - private readonly Elasticsearch7XCluster _cluster; - private IElasticClient _client; - - public Elasticsearch7X(Elasticsearch7XCluster cluster, Elasticsearch7XSetup setup) - { - _cluster = cluster; - _client = cluster.Client; - } - - - [I] public void AssertTemplate() - { - var templateResponse = _client.Indices.GetTemplate(Elasticsearch7XSetup.TemplateName); - templateResponse.TemplateMappings.Should().NotBeEmpty(); - templateResponse.TemplateMappings.Keys.Should().Contain(Elasticsearch7XSetup.TemplateName); - - var template = templateResponse.TemplateMappings[Elasticsearch7XSetup.TemplateName]; - - template.IndexPatterns.Should().Contain(pattern => pattern.StartsWith(Elasticsearch7XSetup.IndexPrefix)); - - } - [I] public void AssertLogs() - { - var refreshed = _client.Indices.Refresh(Elasticsearch7XSetup.IndexPrefix + "*"); - - var search = _client.Search(s => s.Index(Elasticsearch7XSetup.IndexPrefix + "*")); - - // Informational should be filtered - search.Documents.Count().Should().Be(4); - - - } - } -} \ No newline at end of file diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj b/Serilog.Sinks.Elasticsearch.IntegrationTests/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj index a6a57003..fcd70c4e 100644 --- a/Serilog.Sinks.Elasticsearch.IntegrationTests/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj +++ b/Serilog.Sinks.Elasticsearch.IntegrationTests/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj @@ -12,15 +12,20 @@ + + - - + + + + + diff --git a/nuget.config b/nuget.config index 96116f07..a81fc3ec 100644 --- a/nuget.config +++ b/nuget.config @@ -11,5 +11,7 @@ + + \ No newline at end of file diff --git a/serilog-sinks-elasticsearch.sln.DotSettings b/serilog-sinks-elasticsearch.sln.DotSettings index 95859094..e62f9e23 100644 --- a/serilog-sinks-elasticsearch.sln.DotSettings +++ b/serilog-sinks-elasticsearch.sln.DotSettings @@ -1,4 +1,8 @@  + True True + True + True + True True True \ No newline at end of file diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs index 29ccb503..fb3334d0 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs @@ -159,7 +159,7 @@ public class ElasticsearchSinkOptions public IElasticsearchSerializer Serializer { get; set; } /// - /// The connectionpool describing the cluster to write event to + /// The connection pool describing the cluster to write event to /// public IConnectionPool ConnectionPool { get; private set; } @@ -259,7 +259,7 @@ public ElasticsearchSinkOptions() { this.IndexFormat = "logstash-{0:yyyy.MM.dd}"; this.DeadLetterIndexName = "deadletter-{0:yyyy.MM.dd}"; - this.TypeName = "logevent"; + this.TypeName = DefaultTypeName; this.Period = TimeSpan.FromSeconds(2); this.BatchPostingLimit = 50; this.SingleEventSizePostingLimit = null; @@ -274,6 +274,12 @@ public ElasticsearchSinkOptions() this.ConnectionPool = new SingleNodeConnectionPool(_defaultNode); } + /// + /// The default Elasticsearch type name used for Elasticsearch versions prior to 7. + /// As of Elasticsearch 7 and up _type has been removed. + /// + public static string DefaultTypeName { get; } = "logevent"; + /// /// Configures the elasticsearch sink /// From db5ddb13276823cd0e61a690741926fdfbbb7acb Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Mon, 8 Jul 2019 13:20:37 +0200 Subject: [PATCH 06/15] move integration tests under test folder --- serilog-sinks-elasticsearch.sln | 2 +- .../Bootstrap/ClientTestClusterBase.cs | 0 .../Bootstrap/ElasticsearchSinkOptionsFactory.cs | 0 .../Bootstrap/ProxyDetection.cs | 0 .../Bootstrap/SerilogSinkElasticsearchXunitRunOptions.cs | 0 .../Bootstrap/XunitBootstrap.cs | 0 .../Elasticsearch6/Bootstrap/Elasticsearch6XCluster.cs | 0 .../Elasticsearch6/Bootstrap/Elasticsearch6XTestBase.cs | 0 .../Elasticsearch6/Elasticsearch6X.cs | 0 .../Elasticsearch7/Bootstrap/Elasticsearch7XCluster.cs | 0 .../Elasticsearch7/Bootstrap/Elasticsearch7XTestBase.cs | 0 .../Elasticsearch7/Elasticsearch7X.cs | 0 .../Serilog.Sinks.Elasticsearch.IntegrationTests.csproj | 5 +---- 13 files changed, 2 insertions(+), 5 deletions(-) rename {Serilog.Sinks.Elasticsearch.IntegrationTests => test/Serilog.Sinks.Elasticsearch.IntegrationTests}/Bootstrap/ClientTestClusterBase.cs (100%) rename {Serilog.Sinks.Elasticsearch.IntegrationTests => test/Serilog.Sinks.Elasticsearch.IntegrationTests}/Bootstrap/ElasticsearchSinkOptionsFactory.cs (100%) rename {Serilog.Sinks.Elasticsearch.IntegrationTests => test/Serilog.Sinks.Elasticsearch.IntegrationTests}/Bootstrap/ProxyDetection.cs (100%) rename {Serilog.Sinks.Elasticsearch.IntegrationTests => test/Serilog.Sinks.Elasticsearch.IntegrationTests}/Bootstrap/SerilogSinkElasticsearchXunitRunOptions.cs (100%) rename {Serilog.Sinks.Elasticsearch.IntegrationTests => test/Serilog.Sinks.Elasticsearch.IntegrationTests}/Bootstrap/XunitBootstrap.cs (100%) rename {Serilog.Sinks.Elasticsearch.IntegrationTests => test/Serilog.Sinks.Elasticsearch.IntegrationTests}/Elasticsearch6/Bootstrap/Elasticsearch6XCluster.cs (100%) rename {Serilog.Sinks.Elasticsearch.IntegrationTests => test/Serilog.Sinks.Elasticsearch.IntegrationTests}/Elasticsearch6/Bootstrap/Elasticsearch6XTestBase.cs (100%) rename {Serilog.Sinks.Elasticsearch.IntegrationTests => test/Serilog.Sinks.Elasticsearch.IntegrationTests}/Elasticsearch6/Elasticsearch6X.cs (100%) rename {Serilog.Sinks.Elasticsearch.IntegrationTests => test/Serilog.Sinks.Elasticsearch.IntegrationTests}/Elasticsearch7/Bootstrap/Elasticsearch7XCluster.cs (100%) rename {Serilog.Sinks.Elasticsearch.IntegrationTests => test/Serilog.Sinks.Elasticsearch.IntegrationTests}/Elasticsearch7/Bootstrap/Elasticsearch7XTestBase.cs (100%) rename {Serilog.Sinks.Elasticsearch.IntegrationTests => test/Serilog.Sinks.Elasticsearch.IntegrationTests}/Elasticsearch7/Elasticsearch7X.cs (100%) rename {Serilog.Sinks.Elasticsearch.IntegrationTests => test/Serilog.Sinks.Elasticsearch.IntegrationTests}/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj (88%) diff --git a/serilog-sinks-elasticsearch.sln b/serilog-sinks-elasticsearch.sln index bb9350f1..551c394d 100644 --- a/serilog-sinks-elasticsearch.sln +++ b/serilog-sinks-elasticsearch.sln @@ -24,7 +24,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serilog.Sinks.Elasticsearch EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serilog.Formatting.Elasticsearch", "src\Serilog.Formatting.Elasticsearch\Serilog.Formatting.Elasticsearch.csproj", "{0E6D34BF-322A-4803-94D1-355F6D5024BE}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serilog.Sinks.Elasticsearch.IntegrationTests", "Serilog.Sinks.Elasticsearch.IntegrationTests\Serilog.Sinks.Elasticsearch.IntegrationTests.csproj", "{23BC3821-E028-48B4-8F2C-83BB1B8B5525}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serilog.Sinks.Elasticsearch.IntegrationTests", "test\Serilog.Sinks.Elasticsearch.IntegrationTests\Serilog.Sinks.Elasticsearch.IntegrationTests.csproj", "{23BC3821-E028-48B4-8F2C-83BB1B8B5525}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ClientTestClusterBase.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ClientTestClusterBase.cs similarity index 100% rename from Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ClientTestClusterBase.cs rename to test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ClientTestClusterBase.cs diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ElasticsearchSinkOptionsFactory.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ElasticsearchSinkOptionsFactory.cs similarity index 100% rename from Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ElasticsearchSinkOptionsFactory.cs rename to test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ElasticsearchSinkOptionsFactory.cs diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ProxyDetection.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ProxyDetection.cs similarity index 100% rename from Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ProxyDetection.cs rename to test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/ProxyDetection.cs diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/SerilogSinkElasticsearchXunitRunOptions.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/SerilogSinkElasticsearchXunitRunOptions.cs similarity index 100% rename from Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/SerilogSinkElasticsearchXunitRunOptions.cs rename to test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/SerilogSinkElasticsearchXunitRunOptions.cs diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/XunitBootstrap.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/XunitBootstrap.cs similarity index 100% rename from Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/XunitBootstrap.cs rename to test/Serilog.Sinks.Elasticsearch.IntegrationTests/Bootstrap/XunitBootstrap.cs diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Bootstrap/Elasticsearch6XCluster.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Bootstrap/Elasticsearch6XCluster.cs similarity index 100% rename from Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Bootstrap/Elasticsearch6XCluster.cs rename to test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Bootstrap/Elasticsearch6XCluster.cs diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Bootstrap/Elasticsearch6XTestBase.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Bootstrap/Elasticsearch6XTestBase.cs similarity index 100% rename from Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Bootstrap/Elasticsearch6XTestBase.cs rename to test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Bootstrap/Elasticsearch6XTestBase.cs diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6X.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6X.cs similarity index 100% rename from Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6X.cs rename to test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6X.cs diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Bootstrap/Elasticsearch7XCluster.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Bootstrap/Elasticsearch7XCluster.cs similarity index 100% rename from Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Bootstrap/Elasticsearch7XCluster.cs rename to test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Bootstrap/Elasticsearch7XCluster.cs diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Bootstrap/Elasticsearch7XTestBase.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Bootstrap/Elasticsearch7XTestBase.cs similarity index 100% rename from Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Bootstrap/Elasticsearch7XTestBase.cs rename to test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Bootstrap/Elasticsearch7XTestBase.cs diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7X.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7X.cs similarity index 100% rename from Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7X.cs rename to test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7X.cs diff --git a/Serilog.Sinks.Elasticsearch.IntegrationTests/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj similarity index 88% rename from Serilog.Sinks.Elasticsearch.IntegrationTests/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj rename to test/Serilog.Sinks.Elasticsearch.IntegrationTests/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj index fcd70c4e..75df89c8 100644 --- a/Serilog.Sinks.Elasticsearch.IntegrationTests/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj @@ -8,7 +8,7 @@ True - + @@ -25,7 +25,4 @@ - - - From dd3993f90e7221e25a13218f94d178fa5d508857 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Mon, 8 Jul 2019 14:58:21 +0200 Subject: [PATCH 07/15] Add mixed mode support and real version detection --- .../Sinks/ElasticSearch/ElasticSearchSink.cs | 1 + .../ElasticSearchTemplateProvider.cs | 262 ++++++++++-------- .../ElasticSearch/ElasticsearchSinkOptions.cs | 21 ++ .../ElasticSearch/ElasticsearchSinkState.cs | 33 ++- .../Elasticsearch6/Elasticsearch6X.cs | 2 +- .../Elasticsearch6/Elasticsearch6XUsing7X.cs | 72 +++++ .../Elasticsearch7/Elasticsearch7X.cs | 2 +- .../Elasticsearch7/Elasticsearch7XUsing6X.cs | 68 +++++ 8 files changed, 336 insertions(+), 125 deletions(-) create mode 100644 test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6XUsing7X.cs create mode 100644 test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7XUsing6X.cs diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs index d89136fd..b00d4d5b 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs @@ -39,6 +39,7 @@ public ElasticsearchSink(ElasticsearchSinkOptions options) : base(options.BatchPostingLimit, options.Period, options.QueueSizeLimit) { _state = ElasticsearchSinkState.Create(options); + _state.DiscoverClusterVersion(); _state.RegisterTemplateIfNeeded(); } diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs index e247e8b7..6272a563 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using Elasticsearch.Net.Specification.IndicesApi; namespace Serilog.Sinks.Elasticsearch { @@ -31,13 +32,7 @@ public enum AutoRegisterTemplateVersion /// public class ElasticsearchTemplateProvider { - /// - /// - /// - /// - /// - /// - /// + [Obsolete("Use the overload taking ElasticsearchSinkOptions which takes IncludeTypeName into account")] public static object GetTemplate( Dictionary settings, string templateMatchString, @@ -48,96 +43,112 @@ public static object GetTemplate( case AutoRegisterTemplateVersion.ESv5: return GetTemplateESv5(settings, templateMatchString); case AutoRegisterTemplateVersion.ESv6: - return GetTemplateESv6(settings, templateMatchString); + return GetTemplateESv6(null, null, settings, templateMatchString); case AutoRegisterTemplateVersion.ESv7: - return GetTemplateESv7(settings, templateMatchString); + return GetTemplateESv7(null, null, settings, templateMatchString); case AutoRegisterTemplateVersion.ESv2: return GetTemplateESv2(settings, templateMatchString); default: throw new ArgumentOutOfRangeException(nameof(version), version, null); } } + + public static object GetTemplate(ElasticsearchSinkOptions options, + string discoveredVersion, + Dictionary settings, + string templateMatchString, + AutoRegisterTemplateVersion version = AutoRegisterTemplateVersion.ESv2) + { + switch (version) + { + case AutoRegisterTemplateVersion.ESv5: + return GetTemplateESv5(settings, templateMatchString); + case AutoRegisterTemplateVersion.ESv2: + return GetTemplateESv2(settings, templateMatchString); + case AutoRegisterTemplateVersion.ESv6: + return GetTemplateESv6(options, discoveredVersion, settings, templateMatchString); + case AutoRegisterTemplateVersion.ESv7: + return GetTemplateESv7(options, discoveredVersion, settings, templateMatchString); + default: + throw new ArgumentOutOfRangeException(nameof(version), version, null); + } + } - private static object GetTemplateESv7( + private static object GetTemplateESv7(ElasticsearchSinkOptions options, string discoveredVersion, Dictionary settings, string templateMatchString) { - return new + object mappings = new { - index_patterns = new[] { templateMatchString }, - settings = settings, - mappings = new + dynamic_templates = new List { - dynamic_templates = new List + //when you use serilog as an adaptor for third party frameworks + //where you have no control over the log message they typically + //contain {0} ad infinitum, we force numeric property names to + //contain strings by default. { - //when you use serilog as an adaptor for third party frameworks - //where you have no control over the log message they typically - //contain {0} ad infinitum, we force numeric property names to - //contain strings by default. + new { - new + numerics_in_fields = new { - numerics_in_fields = new + path_match = @"fields\.[\d+]$", + match_pattern = "regex", + mapping = new { - path_match = @"fields\.[\d+]$", - match_pattern = "regex", - mapping = new - { - type = "text", - index = true, - norms = false - } + type = "text", + index = true, + norms = false } } - }, + } + }, + { + new { - new + string_fields = new { - string_fields = new + match = "*", + match_mapping_type = "string", + mapping = new { - match = "*", - match_mapping_type = "string", - mapping = new + type = "text", + index = true, + norms = false, + fields = new { - type = "text", - index = true, - norms = false, - fields = new + raw = new { - raw = new - { - type = "keyword", - index = true, - ignore_above = 256 - } + type = "keyword", + index = true, + ignore_above = 256 } } } } } - }, - properties = new Dictionary + } + }, + properties = new Dictionary + { + {"message", new {type = "text", index = true}}, { - {"message", new {type = "text", index = true}}, + "exceptions", new { - "exceptions", new + type = "nested", + properties = new Dictionary { - type = "nested", - properties = new Dictionary + {"Depth", new {type = "integer"}}, + {"RemoteStackIndex", new {type = "integer"}}, + {"HResult", new {type = "integer"}}, + {"StackTraceString", new {type = "text", index = true}}, + {"RemoteStackTraceString", new {type = "text", index = true}}, { - {"Depth", new {type = "integer"}}, - {"RemoteStackIndex", new {type = "integer"}}, - {"HResult", new {type = "integer"}}, - {"StackTraceString", new {type = "text", index = true}}, - {"RemoteStackTraceString", new {type = "text", index = true}}, + "ExceptionMessage", new { - "ExceptionMessage", new + type = "object", + properties = new Dictionary { - type = "object", - properties = new Dictionary - { - {"MemberType", new {type = "integer"}}, - } + {"MemberType", new {type = "integer"}}, } } } @@ -146,91 +157,91 @@ private static object GetTemplateESv7( } } }; + mappings = discoveredVersion?.StartsWith("6.") ?? false ? new { _doc = mappings } : mappings; + + return new + { + index_patterns = new[] {templateMatchString}, + settings = settings, + mappings = mappings + }; } - private static object GetTemplateESv6( + private static object GetTemplateESv6(ElasticsearchSinkOptions options, string discoveredVersion, Dictionary settings, string templateMatchString) { - return new + object mappings = new { - index_patterns = new[] { templateMatchString }, - settings = settings, - mappings = new + dynamic_templates = new List { - _default_ = new + //when you use serilog as an adaptor for third party frameworks + //where you have no control over the log message they typically + //contain {0} ad infinitum, we force numeric property names to + //contain strings by default. { - dynamic_templates = new List + new { - //when you use serilog as an adaptor for third party frameworks - //where you have no control over the log message they typically - //contain {0} ad infinitum, we force numeric property names to - //contain strings by default. + numerics_in_fields = new { - new + path_match = @"fields\.[\d+]$", + match_pattern = "regex", + mapping = new { - numerics_in_fields = new - { - path_match = @"fields\.[\d+]$", - match_pattern = "regex", - mapping = new - { - type = "text", - index = true, - norms = false - } - } + type = "text", + index = true, + norms = false } - }, + } + } + }, + { + new + { + string_fields = new { - new + match = "*", + match_mapping_type = "string", + mapping = new { - string_fields = new + type = "text", + index = true, + norms = false, + fields = new { - match = "*", - match_mapping_type = "string", - mapping = new + raw = new { - type = "text", + type = "keyword", index = true, - norms = false, - fields = new - { - raw = new - { - type = "keyword", - index = true, - ignore_above = 256 - } - } + ignore_above = 256 } } } } - }, - properties = new Dictionary + } + } + }, + properties = new Dictionary + { + {"message", new {type = "text", index = true}}, + { + "exceptions", new { - {"message", new {type = "text", index = true}}, + type = "nested", + properties = new Dictionary { - "exceptions", new + {"Depth", new {type = "integer"}}, + {"RemoteStackIndex", new {type = "integer"}}, + {"HResult", new {type = "integer"}}, + {"StackTraceString", new {type = "text", index = true}}, + {"RemoteStackTraceString", new {type = "text", index = true}}, { - type = "nested", - properties = new Dictionary + "ExceptionMessage", new { - {"Depth", new {type = "integer"}}, - {"RemoteStackIndex", new {type = "integer"}}, - {"HResult", new {type = "integer"}}, - {"StackTraceString", new {type = "text", index = true}}, - {"RemoteStackTraceString", new {type = "text", index = true}}, + type = "object", + properties = new Dictionary { - "ExceptionMessage", new - { - type = "object", - properties = new Dictionary - { - {"MemberType", new {type = "integer"}}, - } - } + {"MemberType", new {type = "integer"}}, } } } @@ -239,6 +250,15 @@ private static object GetTemplateESv6( } } }; + + mappings = discoveredVersion?.StartsWith("7.") ?? false ? (object) new { _doc = mappings} : new { _default_ = mappings}; + + return new + { + index_patterns = new[] { templateMatchString }, + settings = settings, + mappings = mappings + }; } private static object GetTemplateESv5( diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs index fb3334d0..0c450796 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs @@ -280,6 +280,27 @@ public ElasticsearchSinkOptions() /// public static string DefaultTypeName { get; } = "logevent"; + /// + /// Instructs the sink to auto detect the running Elasticsearch version. + /// + /// + /// This information is used to attempt to register an older or newer template + /// + /// + /// + /// + /// Currently supports: + /// + /// + /// + /// + /// Currently supports: + /// - using against Elasticsearch 6.x + /// - using against Elasticsearch 7.x + /// + /// + public bool DetectElasticsearchVersion { get; set; } + /// /// Configures the elasticsearch sink /// diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs index 1b0249e9..ac129a3a 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs @@ -14,9 +14,11 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Text; using System.Text.RegularExpressions; using Elasticsearch.Net; +using Elasticsearch.Net.Specification.CatApi; using Elasticsearch.Net.Specification.IndicesApi; using Serilog.Debugging; using Serilog.Events; @@ -49,7 +51,12 @@ public static ElasticsearchSinkState Create(ElasticsearchSinkOptions options) private readonly string _templateName; private readonly string _templateMatchString; private static readonly Regex IndexFormatRegex = new Regex(@"^(.*)(?:\{0\:.+\})(.*)$"); + private string _discoveredVersion; + public string DiscoveredVersion => _discoveredVersion; + private bool IncludeTypeName => + (DiscoveredVersion?.StartsWith("7.") ?? false) + && _options.AutoRegisterTemplateVersion == AutoRegisterTemplateVersion.ESv6; public ElasticsearchSinkOptions Options => _options; public IElasticLowLevelClient Client => _client; public ITextFormatter Formatter => _formatter; @@ -163,8 +170,11 @@ public void RegisterTemplateIfNeeded() } } - Console.WriteLine(_client.Serializer.SerializeToString(GetTemplateData())); - var result = _client.Indices.PutTemplateForAll(_templateName, GetTempatePostData()); + var result = _client.Indices.PutTemplateForAll(_templateName, GetTempatePostData(), + new PutIndexTemplateRequestParameters + { + IncludeTypeName = IncludeTypeName ? true : (bool?) null + }); if (!result.Success) { @@ -224,10 +234,29 @@ private object GetTemplateData() settings.Add("number_of_replicas", _options.NumberOfReplicas.Value.ToString()); return ElasticsearchTemplateProvider.GetTemplate( + _options, + DiscoveredVersion, settings, _templateMatchString, _options.AutoRegisterTemplateVersion); } + + public void DiscoverClusterVersion() + { + if (!_options.DetectElasticsearchVersion) return; + + var response = _client.Cat.Nodes(new CatNodesRequestParameters() + { + Headers = new [] { "v"} + }); + if (!response.Success) return; + + _discoveredVersion = response.Body.Split(new[] {'\r', '\n'}, StringSplitOptions.RemoveEmptyEntries) + .FirstOrDefault(); + + if (_discoveredVersion?.StartsWith("7.") ?? false) + _options.TypeName = "_doc"; + } } } diff --git a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6X.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6X.cs index 0ac38608..9bb47c2f 100644 --- a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6X.cs +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6X.cs @@ -41,7 +41,7 @@ [I] public void AssertLogs() // ReSharper disable once ClassNeverInstantiated.Global public class SetupSerilog { - public const string IndexPrefix = "logs-6x-"; + public const string IndexPrefix = "logs-6x-default-"; public const string TemplateName = "serilog-logs-6x"; public SetupSerilog() diff --git a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6XUsing7X.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6XUsing7X.cs new file mode 100644 index 00000000..11c8e20d --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6XUsing7X.cs @@ -0,0 +1,72 @@ +using System.Linq; +using Elastic.Xunit.XunitPlumbing; +using FluentAssertions; +using ICSharpCode.SharpZipLib.Core; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch6.Bootstrap; +using Xunit; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch6 +{ + public class Elasticsearch6XUsing7X : Elasticsearch6XTestBase, IClassFixture + { + private readonly SetupSerilog _setup; + + public Elasticsearch6XUsing7X(Elasticsearch6XCluster cluster, SetupSerilog setup) : base(cluster) => _setup = setup; + + [I] public void AssertTemplate() + { + var templateResponse = Client.GetIndexTemplate(t=>t.Name(SetupSerilog.TemplateName)); + templateResponse.TemplateMappings.Should().NotBeEmpty(); + templateResponse.TemplateMappings.Keys.Should().Contain(SetupSerilog.TemplateName); + + var template = templateResponse.TemplateMappings[SetupSerilog.TemplateName]; + + template.IndexPatterns.Should().Contain(pattern => pattern.StartsWith(SetupSerilog.IndexPrefix)); + } + + [I] public void AssertLogs() + { + var refreshed = Client.Refresh(SetupSerilog.IndexPrefix + "*"); + + var search = Client.Search(s => s + .Index(SetupSerilog.IndexPrefix + "*") + .Type("_doc") + ); + + // Informational should be filtered + search.Documents.Count().Should().Be(4); + } + + // ReSharper disable once ClassNeverInstantiated.Global + public class SetupSerilog + { + public const string IndexPrefix = "logs-6x-using-7x-"; + public const string TemplateName = "serilog-logs-6x-using-7x"; + + public SetupSerilog() + { + var loggerConfig = new LoggerConfiguration() + .MinimumLevel.Information() + .WriteTo.ColoredConsole() + .WriteTo.Elasticsearch( + ElasticsearchSinkOptionsFactory.Create(IndexPrefix, TemplateName, o => + { + o.DetectElasticsearchVersion = true; + o.AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7; + o.AutoRegisterTemplate = true; + }) + ); + var logger = loggerConfig.CreateLogger(); + + logger.Information("Hello Information"); + logger.Debug("Hello Debug"); + logger.Warning("Hello Warning"); + logger.Error("Hello Error"); + logger.Fatal("Hello Fatal"); + + logger.Dispose(); + } + } + } +} \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7X.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7X.cs index f025546d..a3d19d70 100644 --- a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7X.cs +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7X.cs @@ -37,7 +37,7 @@ [I] public void AssertLogs() // ReSharper disable once ClassNeverInstantiated.Global public class SetupSerilog { - public const string IndexPrefix = "logs-7x-"; + public const string IndexPrefix = "logs-7x-default-"; public const string TemplateName = "serilog-logs-7x"; public SetupSerilog() diff --git a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7XUsing6X.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7XUsing6X.cs new file mode 100644 index 00000000..69673882 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7XUsing6X.cs @@ -0,0 +1,68 @@ +using System.Linq; +using Elastic.Xunit.XunitPlumbing; +using FluentAssertions; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap; +using Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch7.Bootstrap; +using Xunit; + +namespace Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch7 +{ + public class Elasticsearch7XUsing6X : Elasticsearch7XTestBase, IClassFixture + { + private readonly SetupSerilog _setup; + + public Elasticsearch7XUsing6X(Elasticsearch7XCluster cluster, SetupSerilog setup) : base(cluster) => _setup = setup; + + [I] public void AssertTemplate() + { + var templateResponse = Client.Indices.GetTemplate(SetupSerilog.TemplateName); + templateResponse.TemplateMappings.Should().NotBeEmpty(); + templateResponse.TemplateMappings.Keys.Should().Contain(SetupSerilog.TemplateName); + + var template = templateResponse.TemplateMappings[SetupSerilog.TemplateName]; + + template.IndexPatterns.Should().Contain(pattern => pattern.StartsWith(SetupSerilog.IndexPrefix)); + } + + [I] public void AssertLogs() + { + var refreshed = Client.Indices.Refresh(SetupSerilog.IndexPrefix + "*"); + + var search = Client.Search(s => s.Index(SetupSerilog.IndexPrefix + "*")); + + // Informational should be filtered + search.Documents.Count().Should().Be(4); + } + + // ReSharper disable once ClassNeverInstantiated.Global + public class SetupSerilog + { + public const string IndexPrefix = "logs-7x-using-6x"; + public const string TemplateName = "serilog-logs-7x-using-6x"; + + public SetupSerilog() + { + var loggerConfig = new LoggerConfiguration() + .MinimumLevel.Information() + .WriteTo.ColoredConsole() + .WriteTo.Elasticsearch( + ElasticsearchSinkOptionsFactory.Create(IndexPrefix, TemplateName, o => + { + o.DetectElasticsearchVersion = true; + o.AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv6; + o.AutoRegisterTemplate = true; + }) + ); + using (var logger = loggerConfig.CreateLogger()) + { + logger.Information("Hello Information"); + logger.Debug("Hello Debug"); + logger.Warning("Hello Warning"); + logger.Error("Hello Error"); + logger.Fatal("Hello Fatal"); + } + } + } + + } +} \ No newline at end of file From a07dc2f6b2c4cd0afcc190a0e72c14046549a4e1 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Mon, 8 Jul 2019 14:58:43 +0200 Subject: [PATCH 08/15] clean up namespace --- .../Elasticsearch6/Elasticsearch6XUsing7X.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6XUsing7X.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6XUsing7X.cs index 11c8e20d..9078c9c8 100644 --- a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6XUsing7X.cs +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6XUsing7X.cs @@ -1,7 +1,6 @@ using System.Linq; using Elastic.Xunit.XunitPlumbing; using FluentAssertions; -using ICSharpCode.SharpZipLib.Core; using Serilog.Sinks.Elasticsearch.IntegrationTests.Bootstrap; using Serilog.Sinks.Elasticsearch.IntegrationTests.Elasticsearch6.Bootstrap; using Xunit; From 9177d39c768cd0d12f1ece84fe612423a705e5c2 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Mon, 8 Jul 2019 15:02:15 +0200 Subject: [PATCH 09/15] update unit tests --- .../Templating/template_v6.json | 6 +++--- .../Templating/template_v7.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v6.json b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v6.json index 382f100e..582c2e9b 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v6.json +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v6.json @@ -39,7 +39,7 @@ "properties": { "message": { "type": "text", - "index": "true" + "index": true }, "exceptions": { "type": "nested", @@ -55,11 +55,11 @@ }, "StackTraceString": { "type": "text", - "index": "true" + "index": true }, "RemoteStackTraceString": { "type": "text", - "index": "true" + "index": true }, "ExceptionMessage": { "type": "object", diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v7.json b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v7.json index 585f5323..003ada40 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v7.json +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v7.json @@ -38,7 +38,7 @@ "properties": { "message": { "type": "text", - "index": "true" + "index": true }, "exceptions": { "type": "nested", @@ -54,11 +54,11 @@ }, "StackTraceString": { "type": "text", - "index": "true" + "index": true }, "RemoteStackTraceString": { "type": "text", - "index": "true" + "index": true }, "ExceptionMessage": { "type": "object", From 57f66ad0ef3a4fab29723618f603ae50df6355f7 Mon Sep 17 00:00:00 2001 From: Mpdreamz Date: Mon, 8 Jul 2019 15:56:16 +0200 Subject: [PATCH 10/15] temp rename to Elasticsearch2 --- .../{ElasticSearch => Elasticsearch2}/DurableElasticsearchSink.cs | 0 .../{ElasticSearch => Elasticsearch2}/ElasticsearchLogClient.cs | 0 .../{ElasticSearch => Elasticsearch2}/ElasticsearchLogShipper.cs | 0 .../ElasticsearchPayloadReader.cs | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/{ElasticSearch => Elasticsearch2}/DurableElasticsearchSink.cs (100%) rename src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/{ElasticSearch => Elasticsearch2}/ElasticsearchLogClient.cs (100%) rename src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/{ElasticSearch => Elasticsearch2}/ElasticsearchLogShipper.cs (100%) rename src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/{ElasticSearch => Elasticsearch2}/ElasticsearchPayloadReader.cs (100%) diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/DurableElasticsearchSink.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch2/DurableElasticsearchSink.cs similarity index 100% rename from src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/DurableElasticsearchSink.cs rename to src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch2/DurableElasticsearchSink.cs diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticsearchLogClient.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch2/ElasticsearchLogClient.cs similarity index 100% rename from src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticsearchLogClient.cs rename to src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch2/ElasticsearchLogClient.cs diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticsearchLogShipper.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch2/ElasticsearchLogShipper.cs similarity index 100% rename from src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticsearchLogShipper.cs rename to src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch2/ElasticsearchLogShipper.cs diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticsearchPayloadReader.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch2/ElasticsearchPayloadReader.cs similarity index 100% rename from src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/ElasticSearch/ElasticsearchPayloadReader.cs rename to src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch2/ElasticsearchPayloadReader.cs From 36a1fb7424aa702d82a24fbb64e431bb308017cc Mon Sep 17 00:00:00 2001 From: Mpdreamz Date: Mon, 8 Jul 2019 15:58:14 +0200 Subject: [PATCH 11/15] Rename back to Elasticsearch --- .../{Elasticsearch2 => Elasticsearch}/DurableElasticsearchSink.cs | 0 .../{Elasticsearch2 => Elasticsearch}/ElasticsearchLogClient.cs | 0 .../{Elasticsearch2 => Elasticsearch}/ElasticsearchLogShipper.cs | 0 .../ElasticsearchPayloadReader.cs | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/{Elasticsearch2 => Elasticsearch}/DurableElasticsearchSink.cs (100%) rename src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/{Elasticsearch2 => Elasticsearch}/ElasticsearchLogClient.cs (100%) rename src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/{Elasticsearch2 => Elasticsearch}/ElasticsearchLogShipper.cs (100%) rename src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/{Elasticsearch2 => Elasticsearch}/ElasticsearchPayloadReader.cs (100%) diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch2/DurableElasticsearchSink.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch/DurableElasticsearchSink.cs similarity index 100% rename from src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch2/DurableElasticsearchSink.cs rename to src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch/DurableElasticsearchSink.cs diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch2/ElasticsearchLogClient.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch/ElasticsearchLogClient.cs similarity index 100% rename from src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch2/ElasticsearchLogClient.cs rename to src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch/ElasticsearchLogClient.cs diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch2/ElasticsearchLogShipper.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch/ElasticsearchLogShipper.cs similarity index 100% rename from src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch2/ElasticsearchLogShipper.cs rename to src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch/ElasticsearchLogShipper.cs diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch2/ElasticsearchPayloadReader.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch/ElasticsearchPayloadReader.cs similarity index 100% rename from src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch2/ElasticsearchPayloadReader.cs rename to src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch/ElasticsearchPayloadReader.cs From f3037e33ad8e44d171da1763ee696cb2c09aba7f Mon Sep 17 00:00:00 2001 From: Mpdreamz Date: Mon, 8 Jul 2019 17:04:07 +0200 Subject: [PATCH 12/15] bump testing framework to netcoreapp2.1 since it looks like https://github.com/dotnet/corefx/issues/28156 affects the client --- .../Serilog.Formatting.ElasticSearch.nuspec | 1 + .../Serilog.Sinks.Elasticsearch.csproj | 1 + .../ElasticsearchSinkTestsBase.cs | 26 ++++++++++++++++++- .../Serilog.Sinks.Elasticsearch.Tests.csproj | 12 +++++---- ...dsTemplateHandlesUnavailableServerTests.cs | 2 +- 5 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/Serilog.Formatting.Elasticsearch/Serilog.Formatting.ElasticSearch.nuspec b/src/Serilog.Formatting.Elasticsearch/Serilog.Formatting.ElasticSearch.nuspec index 3047dcd8..50f0444a 100644 --- a/src/Serilog.Formatting.Elasticsearch/Serilog.Formatting.ElasticSearch.nuspec +++ b/src/Serilog.Formatting.Elasticsearch/Serilog.Formatting.ElasticSearch.nuspec @@ -13,6 +13,7 @@ serilog logging elasticsearch formatters + diff --git a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj index 5826a20f..fca22cf1 100644 --- a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj +++ b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj @@ -58,6 +58,7 @@ + diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs b/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs index eb97ad66..e42f11c3 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs @@ -12,6 +12,7 @@ using Serilog.Sinks.Elasticsearch.Tests.Domain; using Nest.JsonNetSerializer; using System.Collections; +using System.Threading; namespace Serilog.Sinks.Elasticsearch.Tests { @@ -135,7 +136,7 @@ Func templateExistReturnCode public override TReturn Request(RequestData requestData) { - MemoryStream ms = new MemoryStream(); + var ms = new MemoryStream(); if (requestData.PostData != null) requestData.PostData.Write(ms, new ConnectionConfiguration()); @@ -155,6 +156,29 @@ public override TReturn Request(RequestData requestData) var responseStream = new MemoryStream(); return ResponseBuilder.ToResponse(requestData, null, this._templateExistReturnCode(), Enumerable.Empty(), responseStream); } + + public override async Task RequestAsync(RequestData requestData, CancellationToken cancellationToken) + { + var ms = new MemoryStream(); + if (requestData.PostData != null) + await requestData.PostData.WriteAsync(ms, new ConnectionConfiguration(), cancellationToken); + + switch (requestData.Method) + { + case HttpMethod.PUT: + _seenHttpPuts.Add(Tuple.Create(requestData.Uri, Encoding.UTF8.GetString(ms.ToArray()))); + break; + case HttpMethod.POST: + _seenHttpPosts.Add(Encoding.UTF8.GetString(ms.ToArray())); + break; + case HttpMethod.HEAD: + _seenHttpHeads.Add(this._templateExistReturnCode()); + break; + } + + var responseStream = new MemoryStream(); + return await ResponseBuilder.ToResponseAsync(requestData, null, this._templateExistReturnCode(), Enumerable.Empty(), responseStream, null, cancellationToken); + } } } } \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj b/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj index a4c073c3..a1f18554 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj @@ -1,12 +1,15 @@  - netcoreapp2.0;net461 + netcoreapp2.1;net461 Serilog.Sinks.Elasticsearch.Tests Serilog.Sinks.Elasticsearch.Tests + + True + latest + True + true - $(PackageTargetFallback);dnxcore50;portable-net45+win8 - 1.1.2 false false false @@ -44,7 +47,6 @@ - @@ -72,7 +74,7 @@ - + $(DefineConstants);DOTNETCORE;PARTIALLY_SERIALIZATION diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SendsTemplateHandlesUnavailableServerTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SendsTemplateHandlesUnavailableServerTests.cs index 76f2e0aa..d549f544 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SendsTemplateHandlesUnavailableServerTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SendsTemplateHandlesUnavailableServerTests.cs @@ -34,7 +34,7 @@ public void Should_write_error_to_self_log() private static ILogger CreateLoggerThatCrashes() { var loggerConfig = new LoggerConfiguration() - .WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri("http://localhost:31234")) + .WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri("http://localhost:9199")) { AutoRegisterTemplate = true, TemplateName = "crash" From 154678a2d293a9c0d2bcbc1c54e823d702ce494d Mon Sep 17 00:00:00 2001 From: Mpdreamz Date: Mon, 8 Jul 2019 17:23:53 +0200 Subject: [PATCH 13/15] add integration tests to ./Build.ps1 --- Build.ps1 | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Build.ps1 b/Build.ps1 index 06c478f5..a6c50387 100644 --- a/Build.ps1 +++ b/Build.ps1 @@ -2,6 +2,7 @@ echo "In directory: $PSScriptRoot" $solution = "serilog-sinks-elasticsearch.sln" $test = "test\\Serilog.Sinks.Elasticsearch.Tests\\Serilog.Sinks.Elasticsearch.Tests.csproj" +$testIntegration = "test\\Serilog.Sinks.Elasticsearch.IntegrationTests\\Serilog.Sinks.Elasticsearch.IntegrationTests.csproj" [string[]]$projects = @( ("src\\Serilog.Sinks.Elasticsearch\\Serilog.Sinks.Elasticsearch.csproj"), ("src\\Serilog.Formatting.Elasticsearch\\Serilog.Formatting.Elasticsearch.csproj") @@ -22,6 +23,16 @@ function Invoke-Build() Write-Output "The tests failed" exit 1 } + + Write-Output "Running integration tests" + # Tee-Object forces console redirection on vstest which magically makes Console.WriteLine works again. + # This allows you to see the console out of Elastic.Xunit while its running + & dotnet test $testIntegration -c Release | Tee-Object -Variable integ + if($LASTEXITCODE -ne 0) + { + Write-Output "The integration tests failed" + exit 1 + } Write-Output "Creating packages" foreach ($project in $projects) From 03f1ab2d3024fe811d0053f847461ed3d8466aa7 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Fri, 12 Jul 2019 20:11:01 +0200 Subject: [PATCH 14/15] remove itemgroups referencing old TFM's' --- .../Serilog.Sinks.Elasticsearch.csproj | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj index 5826a20f..f6279400 100644 --- a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj +++ b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj @@ -28,11 +28,6 @@ false - - 1591;1701;1702 - $(DefineConstants);DURABLE;THREADING_TIMER - - 1591;1701;1702 $(DefineConstants);DURABLE;THREADING_TIMER @@ -43,11 +38,6 @@ $(DefineConstants);DURABLE;THREADING_TIMER;HRESULTS - - 1591;1701;1702 - NU1605 - - From b51629714485253aab8023eee3a74cef0f4c4202 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Fri, 12 Jul 2019 20:12:19 +0200 Subject: [PATCH 15/15] update typo in private method GetTemplatePostData --- .../Sinks/ElasticSearch/ElasticsearchSinkState.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs index ac129a3a..7c775632 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs @@ -170,7 +170,7 @@ public void RegisterTemplateIfNeeded() } } - var result = _client.Indices.PutTemplateForAll(_templateName, GetTempatePostData(), + var result = _client.Indices.PutTemplateForAll(_templateName, GetTemplatePostData(), new PutIndexTemplateRequestParameters { IncludeTypeName = IncludeTypeName ? true : (bool?) null @@ -201,7 +201,7 @@ public void RegisterTemplateIfNeeded() } } - private PostData GetTempatePostData() + private PostData GetTemplatePostData() { //PostData no longer exposes an implict cast from object. Previously it supported that and would inspect the object Type to //determine if it it was a litteral string to write directly or if it was an object that it needed to serialse. Now the onus is