From fa5b0971b32eab2fff76c25657745b7844e1577c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nenad=20Vi=C4=87enti=C4=87?= Date: Mon, 23 Jan 2023 12:38:31 +0100 Subject: [PATCH] Automatically handle `TypeName` parameter for different versions of Elasticsearch (<7, 7 or higher) (#462) * fix file references in the Visual Studio Solution file. * Do not set `TypeName` by default any more. * last version of Elasticsearch that supported user-defined `_type` field - v6.8.x is running out of support on 2022-02-10 (https://www.elastic.co/support/eol) * Automatically handle `ElasticsearchSinkOptions.TypeName` for different versions Elasticsearch (<7, 7+) when `ElasticsearchSinkOptions.DetectElasticsearchVersion` is enabled. * Add unit test for automatic settings of `TypeName` when `DetectElasticsearchVersion` is set to `true`. - Two methods used - instantiate `ElasticsearchSinkState` directly and via `LoggerConfiguration` * Add Elasticsearch v8 template + parsing of Elasticsearch major version to `int` + decision branch for which version of index-template API to use + removal of obsolete `ElasticsearchTemplateProvider.GetTemplate(...)` method overload. * Upgrade to .NET 6.0 and update test-frameworks related NuGet pacakges * Upgrade to Elasticsearch.NET 7.17.5 + handle new "preflight" request https://discuss.elastic.co/t/the-client-is-unable-to-verify-that-the-server-is-elasticsearch-due-to-an-unsuccessful-product-check-call-some-functionality-may-not-be-compatible-if-the-server-is-running-an-unsupported-product/310969/9 "The 7.16 client performs a pre-flight GET request to the root URL of the server before the first request.". * Make `ConnectionStub` a bit more robust . * Use `System.Version` to parse Elasticsearch server version number (similar to what `Elasticsearch.Net` does) * Update NuGet packages * Replace obsolete NuGet package `Serilog.Sinks.ColoredConsole` with `Serilog.Sinks.Console` * Update `Serilog.Sinks.PeriodicBatching` package and reimplent `ElasticsearchSink` so that it does not use obsolete `PeriodicBatchingSink` constructor. * Better handling of Elasticsearch server version number in mocked `ConnectionStub` * Cleanup: refactor to use single `JsonEquals` method. * Turn on `DetectElasticSearchVersion` option by default. Assume version 7 on fallback. * Cleanup: remove unused namespaces * Cleanup: move `ElasticsearchSinkTestsBase` into `Stubs` subfolder. * Refactor: extract `ConnectionStub` into a separate file. * Fix: json comparison in .NET Framework 4.6+ * Run unit-tests on multiple .NET frameworks. * Cleanup: remove unused NUnit runner package. * Use newer, built-in compilation constants. https://learn.microsoft.com/en-us/dotnet/csharp/language-reference/preprocessor-directives#conditional-compilation * Use standard MSBuild property `IsPackable` for clarity. * Cleanup: remove unused package refrence. * Update GitHub actions * docs: updated documentation to reflect changes in behavior of the sink. Co-authored-by: Nenad Vicentic --- CHANGES.md | 20 ++ README.md | 70 ++++++- .../Serilog.Sinks.Elasticsearch.Sample.csproj | 6 +- .../Serilog.Formatting.Elasticsearch.csproj | 2 +- ...gerConfigurationElasticSearchExtensions.cs | 12 +- .../Properties/AssemblyInfo.cs | 2 +- .../Serilog.Sinks.Elasticsearch.csproj | 7 +- .../Elasticsearch/ElasticsearchLogClient.cs | 2 +- .../ElasticsearchPayloadReader.cs | 2 +- .../Sinks/ElasticSearch/ElasticSearchSink.cs | 71 +++---- .../ElasticSearchTemplateProvider.cs | 63 +++--- .../ElasticSearch/ElasticsearchSinkOptions.cs | 12 +- .../ElasticSearch/ElasticsearchSinkState.cs | 84 ++++---- .../ElasticsearchVersionManager.cs | 85 ++++++++ .../Elasticsearch6/Elasticsearch6X.cs | 2 +- .../Elasticsearch6/Elasticsearch6XUsing7X.cs | 2 +- .../Elasticsearch7/Elasticsearch7X.cs | 2 +- .../Elasticsearch7/Elasticsearch7XUsing6X.cs | 2 +- ...inks.Elasticsearch.IntegrationTests.csproj | 21 +- .../BulkActionTests.cs | 3 +- .../CustomIndexTypeNameTests.cs | 9 +- .../ElasticsearchSinkUniformityTestsBase.cs | 5 +- .../Discrepancies/NoSerializerTests.cs | 5 +- .../Domain/BulkAction.cs | 3 - .../ElasticSearchLogShipperTests.cs | 8 +- .../ElasticsearchJsonFormatterTests.cs | 1 - .../ElasticsearchSinkTests.cs | 92 ++++++++ .../ElasticsearchSinkTestsBase.cs | 196 ------------------ .../ExceptionAsJsonObjectFormatterTests.cs | 3 +- .../IndexDeciderTests.cs | 4 +- .../InlineFieldsTests.cs | 2 +- .../Properties/AssemblyInfo.cs | 1 - .../PropertyNameTests.cs | 2 +- .../RealExceptionNoSerializerTests.cs | 2 +- .../RealExceptionTests.cs | 3 +- .../Serilog.Sinks.Elasticsearch.Tests.csproj | 56 +++-- .../Stubs/ConnectionStub.cs | 111 ++++++++++ .../Stubs/ElasticsearchSinkTestsBase.cs | 137 ++++++++++++ ...verVersionHandlesUnavailableServerTests.cs | 1 + .../Templating/DiscoverVersionTests.cs | 3 +- .../DoNotRegisterTemplateIfItExists.cs | 3 +- .../Templating/OverwriteTemplateTests.cs | 9 +- .../Templating/RegisterCustomTemplateTests.cs | 6 +- ...dsTemplateHandlesUnavailableServerTests.cs | 1 + .../Templating/SendsTemplateTests.cs | 27 +-- .../Templating/Sendsv5TemplateTests.cs | 28 +-- .../Templating/Sendsv6TemplateTests.cs | 28 +-- .../Templating/Sendsv7TemplateTests.cs | 28 +-- .../Templating/Sendsv8TemplateTests.cs | 48 +++++ .../Templating/SetElasticsearchSinkOptions.cs | 1 + .../SetFiveReplicasInTemplateTests.cs | 26 +-- .../Templating/SetTwoShardsInTemplateTests.cs | 26 +-- .../SetZeroReplicasInTemplateTests.cs | 26 +-- .../Templating/TemplateMatchTests.cs | 9 +- .../Templating/template_2shards.json | 83 -------- .../Templating/template_v7_no-aliases.json | 77 +++++++ .../{template.json => template_v8.json} | 61 +++--- ... => template_v8_no-aliases_0replicas.json} | 61 +++--- ...on => template_v8_no-aliases_2shards.json} | 63 +++--- ... => template_v8_no-aliases_5replicas.json} | 62 +++--- .../TestDataHelper.cs | 3 - 61 files changed, 1002 insertions(+), 788 deletions(-) create mode 100644 src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchVersionManager.cs create mode 100644 test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTests.cs delete mode 100644 test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs create mode 100644 test/Serilog.Sinks.Elasticsearch.Tests/Stubs/ConnectionStub.cs create mode 100644 test/Serilog.Sinks.Elasticsearch.Tests/Stubs/ElasticsearchSinkTestsBase.cs create mode 100644 test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv8TemplateTests.cs delete mode 100644 test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_2shards.json create mode 100644 test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v7_no-aliases.json rename test/Serilog.Sinks.Elasticsearch.Tests/Templating/{template.json => template_v8.json} (59%) rename test/Serilog.Sinks.Elasticsearch.Tests/Templating/{template_v2.json => template_v8_no-aliases_0replicas.json} (59%) rename test/Serilog.Sinks.Elasticsearch.Tests/Templating/{template_5replicas.json => template_v8_no-aliases_2shards.json} (58%) rename test/Serilog.Sinks.Elasticsearch.Tests/Templating/{template_0replicas.json => template_v8_no-aliases_5replicas.json} (58%) diff --git a/CHANGES.md b/CHANGES.md index e0f1049b..c17efe92 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [9.0.0] - 2023-01-23 + +### Added + - PR #462 + +### Major Changes +- `DetectElasticsearchVersion` is set to `true` by default. +- When `DetectElasticsearchVersion` is set to `false` Elasticsearch version 7 is assumed (as it has broadest wire-compatibility at the moment - v7 and v8) +- When `DetectElasticsearchVersion` is set to `true`, `TypeName` is handled automatically across different versions of Elasticserach (6.x to 8.x). For example, user-defined name will NOT be used on v7 and v8. Also, correct templates endpoint will be picked up. +- Elasticsearch 8.x endpoint for templates is supported (`_index_template`) +- Internal class `ElasticsearchVersionManager` has been added, mainly to handle situations where detection of version fails or when it is disabled. In the case of fallback, sink will assume "default" version 7. +- Elasticsearch.NET client version 7.15.2 (latest version 7, until new `Elastic.Clients.Elasticsearch` 8.x catches up functional parity with 7.x) + +### Other Changes +- Nuget pacakges have been updated (except for the Elasticsearch integration-tests related packages) +- Most of the `ElasticserachSink` functionality has been moved into internal `BatchedElasticsearchSink` class that inherits from `IBatchedLogEventSink`, so it complies with new recommended way of integration with `PeriodicBatchingSink` and we don't use obsolete constructors. +- `ConnectionStub` was moved out of `ElasticsearchSinkTestsBase` and extended. Both are now in `/Stubs` subfolder. Newer versions of Elasticsearch.NET client are now using "pre-flight" request to determine if endpoint is Elasticsearch and if it is indeed between 6.x and 8.x. `ConnectionStub` had to accommodate for that. +- Unit tests have been fixed/added accordingly, running on multiple target frameworks (`net6`, `net7` and `net48`). +- Built-in .NET SDK conditional compilation symbols are now used (e.g NETFRAMEWORK). + ## [9.0.0] - 2022-04-29 ### Fixed diff --git a/README.md b/README.md index f68764e2..282d9069 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ The Serilog Elasticsearch sink project is a sink (basically a writer) for the Se * Starting from version 3, compatible with Elasticsearch 2. * Version 6.x supports the new Elasticsearch.net version 6.x library. * From version 8.x there is support for Elasticsearch.net version 7. +* From version 9.x there is support for Elasticsearch.net version 8. Version detection is enabled by default, in which case `TypeName` is handled automatically across major versions 6, 7 and 8. ## Quick start @@ -40,16 +41,80 @@ The Serilog Elasticsearch sink project is a sink (basically a writer) for the Se Install-Package serilog.sinks.elasticsearch ``` -Register the sink in code or using the appSettings reader (from v2.0.42+) as shown below. Make sure to specify the version of ES you are targeting. Be aware that the AutoRegisterTemplate option will not overwrite an existing template. +Simplest way to register this sink is to use default configuration: + +```csharp +var loggerConfig = new LoggerConfiguration() + .WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri("http://localhost:9200")); +``` + +Or, if using .NET Core and `Serilog.Settings.Configuration` Nuget package and `appsettings.json`, default configuration would look like this: + +```json +{ + "Serilog": { + "Using": [ "Serilog.Sinks.Elasticsearch" ], + "MinimumLevel": "Warning", + "WriteTo": [ + { + "Name": "Elasticsearch", + "Args": { + "nodeUris": "http://localhost:9200" + } + } + ], + "Enrich": [ "FromLogContext", "WithMachineName" ], + "Properties": { + "Application": "ImmoValuation.Swv - Web" + } + } +} +``` + +More elaborate configuration, using additional Nuget packages (e.g. `Serilog.Enrichers.Environment`) would look like: + +```json +{ + "Serilog": { + "Using": [ "Serilog.Sinks.Elasticsearch" ], + "MinimumLevel": "Warning", + "WriteTo": [ + { + "Name": "Elasticsearch", + "Args": { + "nodeUris": "http://localhost:9200" + } + } + ], + "Enrich": [ "FromLogContext", "WithMachineName" ], + "Properties": { + "Application": "My app" + } + } +} +``` + +This way the sink will detect version of Elasticsearch server (`DetectElasticsearchVersion` is set to `true` by default) and handle `TypeName` behavior correctly, based on the server version (6.x, 7.x or 8.x). + +### Disable detection of Elasticsearch server version + +Alternatively, `DetectElasticsearchVersion` can be set to `false` and certain option can be configured manually. In that case, the sink will assume version 7 of Elasticsearch, but options will be ignored due to a potential version incompatibility. + +For example, you can configure the sink to force registeration of v6 index template. Be aware that the AutoRegisterTemplate option will not overwrite an existing template. ```csharp var loggerConfig = new LoggerConfiguration() .WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri("http://localhost:9200") ){ + DetectElasticsearchVersion = false, AutoRegisterTemplate = true, AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv6 }); ``` +### Configurable properties + +Besides a registration of the sink in the code, it is possible to register it using appSettings reader (from v2.0.42+) reader (from v2.0.42+) as shown below. + This example shows the options that are currently available when using the appSettings reader. ```xml @@ -75,7 +140,8 @@ This example shows the options that are currently available when using the appSe - + + diff --git a/sample/Serilog.Sinks.Elasticsearch.Sample/Serilog.Sinks.Elasticsearch.Sample.csproj b/sample/Serilog.Sinks.Elasticsearch.Sample/Serilog.Sinks.Elasticsearch.Sample.csproj index 59ed04fe..2e04d39f 100644 --- a/sample/Serilog.Sinks.Elasticsearch.Sample/Serilog.Sinks.Elasticsearch.Sample.csproj +++ b/sample/Serilog.Sinks.Elasticsearch.Sample/Serilog.Sinks.Elasticsearch.Sample.csproj @@ -2,7 +2,7 @@ Exe - netcoreapp3.1 + net6.0 false @@ -11,8 +11,8 @@ - - + + diff --git a/src/Serilog.Formatting.Elasticsearch/Serilog.Formatting.Elasticsearch.csproj b/src/Serilog.Formatting.Elasticsearch/Serilog.Formatting.Elasticsearch.csproj index 7c7af909..dba7a947 100644 --- a/src/Serilog.Formatting.Elasticsearch/Serilog.Formatting.Elasticsearch.csproj +++ b/src/Serilog.Formatting.Elasticsearch/Serilog.Formatting.Elasticsearch.csproj @@ -28,7 +28,7 @@ - + diff --git a/src/Serilog.Sinks.Elasticsearch/LoggerConfigurationElasticSearchExtensions.cs b/src/Serilog.Sinks.Elasticsearch/LoggerConfigurationElasticSearchExtensions.cs index 3f9159c0..f5173500 100644 --- a/src/Serilog.Sinks.Elasticsearch/LoggerConfigurationElasticSearchExtensions.cs +++ b/src/Serilog.Sinks.Elasticsearch/LoggerConfigurationElasticSearchExtensions.cs @@ -143,6 +143,7 @@ public static LoggerConfiguration Elasticsearch( /// Sink to use when Elasticsearch is unable to accept the events. This is optionally and depends on the EmitEventFailure setting. /// The maximum length of an event allowed to be posted to Elasticsearch.default null /// Add custom elasticsearch settings to the template + /// Turns on detection of elasticsearch version via background HTTP call. This will also set `TypeName` automatically, according to the version of Elasticsearch. /// Configures the OpType being used when inserting document in batch. Must be set to create for data streams. /// LoggerConfiguration object /// is . @@ -151,7 +152,7 @@ public static LoggerConfiguration Elasticsearch( string nodeUris, string indexFormat = null, string templateName = null, - string typeName = "logevent", + string typeName = null, int batchPostingLimit = 50, int period = 2, bool inlineFields = false, @@ -166,7 +167,7 @@ public static LoggerConfiguration Elasticsearch( int queueSizeLimit = 100000, string pipelineName = null, bool autoRegisterTemplate = false, - AutoRegisterTemplateVersion autoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv2, + AutoRegisterTemplateVersion? autoRegisterTemplateVersion = null, bool overwriteTemplate = false, RegisterTemplateRecovery registerTemplateFailure = RegisterTemplateRecovery.IndexAnyway, string deadLetterIndexName = null, @@ -182,7 +183,8 @@ public static LoggerConfiguration Elasticsearch( long? singleEventSizePostingLimit = null, int? bufferFileCountLimit = null, Dictionary templateCustomSettings = null, - ElasticOpType batchAction = ElasticOpType.Index) + ElasticOpType batchAction = ElasticOpType.Index, + bool detectElasticsearchVersion = true) { if (string.IsNullOrEmpty(nodeUris)) throw new ArgumentNullException(nameof(nodeUris), "No Elasticsearch node(s) specified."); @@ -204,8 +206,6 @@ public static LoggerConfiguration Elasticsearch( options.TemplateName = templateName; } - options.TypeName = !string.IsNullOrWhiteSpace(typeName) ? typeName : null; - options.BatchPostingLimit = batchPostingLimit; options.BatchAction = batchAction; options.SingleEventSizePostingLimit = singleEventSizePostingLimit; @@ -272,6 +272,8 @@ public static LoggerConfiguration Elasticsearch( options.TemplateCustomSettings = templateCustomSettings; + options.DetectElasticsearchVersion = detectElasticsearchVersion; + return Elasticsearch(loggerSinkConfiguration, options); } } diff --git a/src/Serilog.Sinks.Elasticsearch/Properties/AssemblyInfo.cs b/src/Serilog.Sinks.Elasticsearch/Properties/AssemblyInfo.cs index c5f05dfe..cd4afa34 100644 --- a/src/Serilog.Sinks.Elasticsearch/Properties/AssemblyInfo.cs +++ b/src/Serilog.Sinks.Elasticsearch/Properties/AssemblyInfo.cs @@ -5,7 +5,7 @@ [assembly: AssemblyDescription("Serilog sink for Elasticsearch")] [assembly: AssemblyCopyright("Copyright © Serilog Contributors 2018")] -[assembly: InternalsVisibleTo("Serilog.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100fb8d13fd344a1c" + +[assembly: InternalsVisibleTo("Serilog.Sinks.Elasticsearch.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100fb8d13fd344a1c" + "6fe0fe83ef33c1080bf30690765bc6eb0df26ebfdf8f21670c64265b30db09f73a0dea5b3db4c9" + "d18dbf6d5a25af5ce9016f281014d79dc3b4201ac646c451830fc7e61a2dfd633d34c39f87b818" + "94191652df5ac63cc40c77f3542f702bda692e6e8a9158353df189007a49da0f3cfd55eb250066" + diff --git a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj index 889e0a1d..953e5aba 100644 --- a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj +++ b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj @@ -4,6 +4,7 @@ Michiel van Oudheusden, Martijn Laarman, Mogens Heller Grabe, Serilog Contributors netstandard2.0 true + latest true Serilog.Sinks.Elasticsearch ../../assets/Serilog.snk @@ -35,11 +36,11 @@ - + - - + + diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch/ElasticsearchLogClient.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch/ElasticsearchLogClient.cs index b3573e77..609d9590 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch/ElasticsearchLogClient.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/Durable/Elasticsearch/ElasticsearchLogClient.cs @@ -86,7 +86,7 @@ private InvalidResult GetInvalidPayloadAsync(DynamicResponse baseResult, List FinishPayLoad() protected override void AddToPayLoad(string nextLine) { var indexName = _getIndexForEvent(nextLine, _date); - var action = ElasticsearchSink.CreateElasticAction( + var action = BatchedElasticsearchSink.CreateElasticAction( opType: _elasticOpType, indexName: indexName, pipelineName: _pipelineName, id: _count + "_" + Guid.NewGuid(), diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs index a0beddee..52c0871d 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs @@ -25,26 +25,39 @@ namespace Serilog.Sinks.Elasticsearch { + public sealed class ElasticsearchSink : PeriodicBatchingSink + { + public ElasticsearchSink(ElasticsearchSinkOptions options) + : this( + new BatchedElasticsearchSink(options), + new PeriodicBatchingSinkOptions + { + BatchSizeLimit = options.BatchPostingLimit, + Period = options.Period, + EagerlyEmitFirstEvent = true, + QueueLimit = (options.QueueSizeLimit == -1) ? null : new int?(options.QueueSizeLimit) + } + ) + { + } + + private ElasticsearchSink(IBatchedLogEventSink batchedSink, PeriodicBatchingSinkOptions options) + : base(batchedSink, options) + { + } + } /// /// Writes log events as documents to ElasticSearch. /// - public class ElasticsearchSink : PeriodicBatchingSink + internal sealed class BatchedElasticsearchSink : IBatchedLogEventSink { - private readonly ElasticsearchSinkState _state; - /// - /// Creates a new ElasticsearchSink instance with the provided options - /// - /// Options configuring how the sink behaves, may NOT be null - public ElasticsearchSink(ElasticsearchSinkOptions options) - : base(options.BatchPostingLimit, options.Period, options.QueueSizeLimit) + public BatchedElasticsearchSink(ElasticsearchSinkOptions options) { _state = ElasticsearchSinkState.Create(options); - _state.DiscoverClusterVersion(); _state.RegisterTemplateIfNeeded(); } - /// /// Emit a batch of log events, running to completion synchronously. /// @@ -54,7 +67,7 @@ public ElasticsearchSink(ElasticsearchSinkOptions options) /// or , /// not both. /// - protected override async Task EmitBatchAsync(IEnumerable events) + public async Task EmitBatchAsync(IEnumerable events) { DynamicResponse result; @@ -71,19 +84,9 @@ protected override async Task EmitBatchAsync(IEnumerable events) HandleResponse(events, result); } - /// - /// Emit a batch of log events, running to completion synchronously. - /// - /// The events to emit. - /// Response from Elasticsearch - protected virtual Task EmitBatchCheckedAsync(IEnumerable events) where T : class, IElasticsearchResponse, new() + public Task OnEmptyBatchAsync() { - // ReSharper disable PossibleMultipleEnumeration - if (events == null || !events.Any()) - return Task.FromResult(default(T)); - - var payload = CreatePayload(events); - return _state.Client.BulkAsync(PostData.MultiJson(payload)); + return Task.CompletedTask; } /// @@ -91,14 +94,14 @@ protected override async Task EmitBatchAsync(IEnumerable events) /// /// The events to emit. /// Response from Elasticsearch - protected virtual T EmitBatchChecked(IEnumerable events) where T : class, IElasticsearchResponse, new() + private Task EmitBatchCheckedAsync(IEnumerable events) where T : class, IElasticsearchResponse, new() { // ReSharper disable PossibleMultipleEnumeration if (events == null || !events.Any()) - return null; + return Task.FromResult(default(T)); var payload = CreatePayload(events); - return _state.Client.Bulk(PostData.MultiJson(payload)); + return _state.Client.BulkAsync(PostData.MultiJson(payload)); } /// @@ -106,7 +109,7 @@ protected override async Task EmitBatchAsync(IEnumerable events) /// /// /// - protected virtual void HandleException(Exception ex, IEnumerable events) + private void HandleException(Exception ex, IEnumerable events) { if (_state.Options.EmitEventFailure.HasFlag(EmitEventFailureHandling.WriteToSelfLog)) { @@ -153,18 +156,6 @@ protected virtual void HandleException(Exception ex, IEnumerable event throw ex; } - // Helper function: checks if a given dynamic member / dictionary key exists at runtime - private static bool HasProperty(dynamic settings, string name) - { - if (settings is IDictionary) - return ((IDictionary)settings).ContainsKey(name); - - if (settings is System.Dynamic.DynamicObject) - return ((System.Dynamic.DynamicObject)settings).GetDynamicMemberNames().Contains(name); - - return settings.GetType().GetProperty(name) != null; - } - private IEnumerable CreatePayload(IEnumerable events) { if (!_state.TemplateRegistrationSuccess && _state.Options.RegisterTemplateFailure == RegisterTemplateRecovery.FailSink) @@ -280,7 +271,7 @@ internal static object CreateElasticAction(ElasticOpType opType, string indexNam : new ElasticIndexAction(actionPayload); return action; } - + sealed class ElasticCreateAction { public ElasticCreateAction(ElasticActionPayload payload) diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs index c69c19ca..d31e8451 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchTemplateProvider.cs @@ -24,57 +24,61 @@ public enum AutoRegisterTemplateVersion /// /// Elasticsearch version >= version 7.0 /// - ESv7 = 3 + ESv7 = 3, + /// + /// Elasticsearch version >= version 8.0 + /// + ESv8 = 4 } /// /// /// public class ElasticsearchTemplateProvider - { - [Obsolete("Use the overload taking ElasticsearchSinkOptions which takes IncludeTypeName into account")] - public static object GetTemplate( + { + public static object GetTemplate(ElasticsearchSinkOptions options, + int discoveredMajorVersion, Dictionary settings, string templateMatchString, AutoRegisterTemplateVersion version = AutoRegisterTemplateVersion.ESv2) { switch (version) { + case AutoRegisterTemplateVersion.ESv8: + return GetTemplateESv8(options, discoveredMajorVersion, settings, templateMatchString); + case AutoRegisterTemplateVersion.ESv7: + return GetTemplateESv7(options, discoveredMajorVersion, settings, templateMatchString); + case AutoRegisterTemplateVersion.ESv6: + return GetTemplateESv6(options, discoveredMajorVersion, settings, templateMatchString); case AutoRegisterTemplateVersion.ESv5: return GetTemplateESv5(settings, templateMatchString); - case AutoRegisterTemplateVersion.ESv6: - return GetTemplateESv6(null, null, settings, templateMatchString); - case AutoRegisterTemplateVersion.ESv7: - return GetTemplateESv7(null, null, settings, templateMatchString); case AutoRegisterTemplateVersion.ESv2: return GetTemplateESv2(settings, templateMatchString); default: throw new ArgumentOutOfRangeException(nameof(version), version, null); } } - - public static object GetTemplate(ElasticsearchSinkOptions options, - string discoveredVersion, + + private static object GetTemplateESv8(ElasticsearchSinkOptions options, int discoveredMajorVersion, Dictionary settings, - string templateMatchString, - AutoRegisterTemplateVersion version = AutoRegisterTemplateVersion.ESv2) + string templateMatchString) { - switch (version) + dynamic templateV7 = GetTemplateESv7(options, discoveredMajorVersion, settings, templateMatchString); + + // wrap settings, mappings and aliases into template property. + return new { - case AutoRegisterTemplateVersion.ESv5: - return GetTemplateESv5(settings, templateMatchString); - case AutoRegisterTemplateVersion.ESv2: - return GetTemplateESv2(settings, templateMatchString); - case AutoRegisterTemplateVersion.ESv6: - return GetTemplateESv6(options, discoveredVersion, settings, templateMatchString); - case AutoRegisterTemplateVersion.ESv7: - return GetTemplateESv7(options, discoveredVersion, settings, templateMatchString); - default: - throw new ArgumentOutOfRangeException(nameof(version), version, null); - } + index_patterns = templateV7.index_patterns, + template = new + { + settings = templateV7.settings, + mappings = templateV7.mappings, + aliases = templateV7.aliases + } + }; } - private static object GetTemplateESv7(ElasticsearchSinkOptions options, string discoveredVersion, + private static object GetTemplateESv7(ElasticsearchSinkOptions options, int discoveredMajorVersion, Dictionary settings, string templateMatchString) { @@ -157,7 +161,7 @@ private static object GetTemplateESv7(ElasticsearchSinkOptions options, string d } } }; - mappings = discoveredVersion?.StartsWith("6.") ?? false ? new { _doc = mappings } : mappings; + mappings = discoveredMajorVersion == 6 ? new { _doc = mappings } : mappings; Dictionary aliases = new Dictionary(); @@ -170,6 +174,7 @@ private static object GetTemplateESv7(ElasticsearchSinkOptions options, string d aliases.Add(alias, new object()); } + return new { index_patterns = new[] { templateMatchString }, @@ -179,7 +184,7 @@ private static object GetTemplateESv7(ElasticsearchSinkOptions options, string d }; } - private static object GetTemplateESv6(ElasticsearchSinkOptions options, string discoveredVersion, + private static object GetTemplateESv6(ElasticsearchSinkOptions options, int discoveredMajorVersion, Dictionary settings, string templateMatchString) { @@ -263,7 +268,7 @@ private static object GetTemplateESv6(ElasticsearchSinkOptions options, string d } }; - mappings = discoveredVersion?.StartsWith("7.") ?? false ? (object) new { _doc = mappings} : new { _default_ = mappings}; + mappings = discoveredMajorVersion == 7 ? (object) new { _doc = mappings} : new { _default_ = mappings}; return new { diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs index 25615e33..f759be2e 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs @@ -38,9 +38,9 @@ public class ElasticsearchSinkOptions /// /// When using the feature, this allows to set the Elasticsearch version. Depending on the - /// version, a template will be selected. Defaults to pre 5.0. + /// version, a template will be selected. Defaults to 7.0. /// - public AutoRegisterTemplateVersion AutoRegisterTemplateVersion { get; set; } + public AutoRegisterTemplateVersion? AutoRegisterTemplateVersion { get; set; } /// /// Specifies the option on how to handle failures when writing the template to Elasticsearch. This is only applicable when using the AutoRegisterTemplate option. @@ -110,7 +110,7 @@ public class ElasticsearchSinkOptions public string DeadLetterIndexName { get; set; } /// - /// The default elasticsearch type name to use for the log events. Defaults to: "_doc". + /// The default elasticsearch type name to use for the log events. Defaults to: null. /// public string TypeName { get; set; } @@ -288,7 +288,6 @@ public ElasticsearchSinkOptions() { this.IndexFormat = "logstash-{0:yyyy.MM.dd}"; this.DeadLetterIndexName = "deadletter-{0:yyyy.MM.dd}"; - this.TypeName = DefaultTypeName; this.Period = TimeSpan.FromSeconds(2); this.BatchPostingLimit = 50; this.SingleEventSizePostingLimit = null; @@ -308,13 +307,14 @@ public ElasticsearchSinkOptions() /// The default Elasticsearch type name used for Elasticsearch versions prior to 7. /// As of Elasticsearch 7 and up _type has been removed. /// - public static string DefaultTypeName { get; } = "_doc"; + public static string DefaultTypeName { get; } = "logevent"; /// /// Instructs the sink to auto detect the running Elasticsearch version. /// /// /// This information is used to attempt to register an older or newer template + /// and to decide which version of index-template API to use. /// /// /// @@ -329,7 +329,7 @@ public ElasticsearchSinkOptions() /// - using against Elasticsearch 7.x /// /// - public bool DetectElasticsearchVersion { get; set; } + public bool DetectElasticsearchVersion { get; set; } = true; /// /// Configures the elasticsearch sink diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs index c304a8f1..ea5de02a 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs @@ -14,16 +14,14 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Text.RegularExpressions; using Elasticsearch.Net; -using Elasticsearch.Net.Specification.CatApi; using Elasticsearch.Net.Specification.IndicesApi; using Serilog.Debugging; using Serilog.Events; using Serilog.Formatting; using Serilog.Formatting.Elasticsearch; +using Serilog.Sinks.Elasticsearch.Sinks.ElasticSearch; namespace Serilog.Sinks.Elasticsearch { @@ -51,12 +49,11 @@ public static ElasticsearchSinkState Create(ElasticsearchSinkOptions options) private readonly string _templateName; private readonly string _templateMatchString; private static readonly Regex IndexFormatRegex = new Regex(@"^(.*)(?:\{0\:.+\})(.*)$"); - private string _discoveredVersion; - public string DiscoveredVersion => _discoveredVersion; - private bool IncludeTypeName => - (DiscoveredVersion?.StartsWith("7.") ?? false) - && _options.AutoRegisterTemplateVersion == AutoRegisterTemplateVersion.ESv6; + private readonly ElasticsearchVersionManager _versionManager; + + private bool IncludeTypeName => _versionManager.EffectiveVersion.Major >= 7; + public ElasticsearchSinkOptions Options => _options; public IElasticLowLevelClient Client => _client; public ITextFormatter Formatter => _formatter; @@ -83,7 +80,6 @@ private ElasticsearchSinkState(ElasticsearchSinkOptions options) _options = options; - var configuration = new ConnectionConfiguration(options.ConnectionPool, options.Connection, options.Serializer) .RequestTimeout(options.ConnectionTimeout); @@ -100,6 +96,16 @@ private ElasticsearchSinkState(ElasticsearchSinkOptions options) _registerTemplateOnStartup = options.AutoRegisterTemplate; TemplateRegistrationSuccess = !_registerTemplateOnStartup; + + _versionManager = new ElasticsearchVersionManager(options.DetectElasticsearchVersion, _client); + + // Resolve typeName + if (_versionManager.EffectiveVersion.Major < 7) + _options.TypeName = string.IsNullOrWhiteSpace(_options.TypeName) + ? ElasticsearchSinkOptions.DefaultTypeName // "logevent" + : _options.TypeName; + else + _options.TypeName = null; } public static ITextFormatter CreateDefaultFormatter(ElasticsearchSinkOptions options) @@ -166,11 +172,20 @@ public void RegisterTemplateIfNeeded() } } - var result = _client.Indices.PutTemplateForAll(_templateName, GetTemplatePostData(), - new PutIndexTemplateRequestParameters - { - IncludeTypeName = IncludeTypeName ? true : (bool?)null - }); + StringResponse result; + if (_versionManager.EffectiveVersion.Major < 8) + { + result = _client.Indices.PutTemplateForAll(_templateName, GetTemplatePostData(), + new PutIndexTemplateRequestParameters + { + IncludeTypeName = IncludeTypeName ? true : (bool?)null + }); + } + else + { + // Default to version 8 API + result = _client.Indices.PutTemplateV2ForAll(_templateName, GetTemplatePostData()); + } if (!result.Success) { @@ -229,38 +244,25 @@ private object GetTemplateData() if (_options.NumberOfReplicas.HasValue && !settings.ContainsKey("number_of_replicas")) settings.Add("number_of_replicas", _options.NumberOfReplicas.Value.ToString()); + var effectiveTemplateVerson = + _options.AutoRegisterTemplateVersion ?? + _versionManager.EffectiveVersion.Major switch + { + >= 8 => AutoRegisterTemplateVersion.ESv8, + 7 => AutoRegisterTemplateVersion.ESv7, + 6 => AutoRegisterTemplateVersion.ESv6, + 5 => AutoRegisterTemplateVersion.ESv5, + 2 => AutoRegisterTemplateVersion.ESv2, + _ => throw new NotSupportedException() + }; + return ElasticsearchTemplateProvider.GetTemplate( _options, - DiscoveredVersion, + _versionManager.EffectiveVersion.Major, settings, _templateMatchString, - _options.AutoRegisterTemplateVersion); + effectiveTemplateVerson); } - - public void DiscoverClusterVersion() - { - if (!_options.DetectElasticsearchVersion) return; - - try - { - - var response = _client.Cat.Nodes(new CatNodesRequestParameters() - { - Headers = new[] { "v" } - }); - if (!response.Success) return; - - _discoveredVersion = response.Body.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries) - .FirstOrDefault(); - - if (_discoveredVersion?.StartsWith("7.") ?? false) - _options.TypeName = "_doc"; - } - catch (Exception ex) - { - SelfLog.WriteLine("Failed to discover the cluster version. {0}", ex); - } - } } } diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchVersionManager.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchVersionManager.cs new file mode 100644 index 00000000..7322c328 --- /dev/null +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchVersionManager.cs @@ -0,0 +1,85 @@ +#nullable enable +using Elasticsearch.Net; +using Elasticsearch.Net.Specification.CatApi; +using Serilog.Debugging; +using System; +using System.Linq; + +namespace Serilog.Sinks.Elasticsearch.Sinks.ElasticSearch +{ + /// + /// Encapsulates detection of Elasticsearch version + /// and fallback in case of detection failiure. + /// + internal class ElasticsearchVersionManager + { + private readonly bool _detectElasticsearchVersion; + private readonly IElasticLowLevelClient _client; + + /// + /// We are defaulting to version 7.17.0 + /// as currently supported versions are 7 and 8, + /// while version 8 retains wire backward compatibility with 7.17.0 + /// and index backward compatibility with 7.0.0 + /// + public readonly Version DefaultVersion = new(7, 17); + public Version? DetectedVersion { get; private set; } + public bool DetectionAttempted { get; private set; } + + public ElasticsearchVersionManager( + bool detectElasticsearchVersion, + IElasticLowLevelClient client) + { + _detectElasticsearchVersion = detectElasticsearchVersion; + _client = client ?? throw new ArgumentNullException(nameof(client)); + } + + public Version EffectiveVersion + { + get + { + if (DetectedVersion is not null) + return DetectedVersion; + + if (_detectElasticsearchVersion == false + || DetectionAttempted == true) + return DefaultVersion; + + // Attemp once + DetectedVersion = DiscoverClusterVersion(); + + return DetectedVersion ?? DefaultVersion; + } + } + + internal Version? DiscoverClusterVersion() + { + try + { + var response = _client.Cat.Nodes(new CatNodesRequestParameters() + { + Headers = new[] { "v" } + }); + if (!response.Success) return null; + + var discoveredVersion = response.Body.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries) + .FirstOrDefault(); + + if (discoveredVersion == null) + return null; + + return new Version(discoveredVersion); + + } + catch (Exception ex) + { + SelfLog.WriteLine("Failed to discover the cluster version. {0}", ex); + return null; + } + finally + { + DetectionAttempted = true; + } + } + } +} diff --git a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6X.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6X.cs index f120fa46..e0110ab9 100644 --- a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6X.cs +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6X.cs @@ -48,7 +48,7 @@ public SetupSerilog() { var loggerConfig = new LoggerConfiguration() .MinimumLevel.Information() - .WriteTo.ColoredConsole() + .WriteTo.Console() .WriteTo.Elasticsearch( ElasticsearchSinkOptionsFactory.Create(IndexPrefix, TemplateName, o => { diff --git a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6XUsing7X.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6XUsing7X.cs index f2a38a82..26cbec10 100644 --- a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6XUsing7X.cs +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch6/Elasticsearch6XUsing7X.cs @@ -47,7 +47,7 @@ public SetupSerilog() { var loggerConfig = new LoggerConfiguration() .MinimumLevel.Information() - .WriteTo.ColoredConsole() + .WriteTo.Console() .WriteTo.Elasticsearch( ElasticsearchSinkOptionsFactory.Create(IndexPrefix, TemplateName, o => { diff --git a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7X.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7X.cs index dea4b051..5a3ff54d 100644 --- a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7X.cs +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7X.cs @@ -44,7 +44,7 @@ public SetupSerilog() { var loggerConfig = new LoggerConfiguration() .MinimumLevel.Information() - .WriteTo.ColoredConsole() + .WriteTo.Console() .WriteTo.Elasticsearch( ElasticsearchSinkOptionsFactory.Create(IndexPrefix, TemplateName, o => { diff --git a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7XUsing6X.cs b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7XUsing6X.cs index 6ec69890..274506b6 100644 --- a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7XUsing6X.cs +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Elasticsearch7/Elasticsearch7XUsing6X.cs @@ -44,7 +44,7 @@ public SetupSerilog() { var loggerConfig = new LoggerConfiguration() .MinimumLevel.Information() - .WriteTo.ColoredConsole() + .WriteTo.Console() .WriteTo.Elasticsearch( ElasticsearchSinkOptionsFactory.Create(IndexPrefix, TemplateName, o => { diff --git a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj index e2973bdc..e47c7f22 100644 --- a/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj +++ b/test/Serilog.Sinks.Elasticsearch.IntegrationTests/Serilog.Sinks.Elasticsearch.IntegrationTests.csproj @@ -1,11 +1,12 @@  - netcoreapp3.1 + net6.0 $(NoWarn);xUnit1013 True latest True + false true @@ -15,21 +16,21 @@ + - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - + + - - - - - + + + + diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/BulkActionTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/BulkActionTests.cs index 143ca14f..0cbd17dd 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/BulkActionTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/BulkActionTests.cs @@ -3,6 +3,7 @@ using FluentAssertions; using Serilog.Events; using Serilog.Parsing; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; using Xunit; namespace Serilog.Sinks.Elasticsearch.Tests @@ -30,7 +31,7 @@ public void DefaultBulkActionV7() public void BulkActionV7OverrideTypeName() { _options.IndexFormat = "logs"; - _options.TypeName = "logevent"; // This is the default value when creating the sink via configuration + _options.TypeName = null; // This is the default value, starting v9.0.0 _options.AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7; _options.PipelineName = null; using (var sink = new ElasticsearchSink(_options)) diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/CustomIndexTypeNameTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/CustomIndexTypeNameTests.cs index d3f44b53..c870334a 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/CustomIndexTypeNameTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/CustomIndexTypeNameTests.cs @@ -1,16 +1,21 @@ using System; using System.Collections.Generic; -using System.Linq; using FluentAssertions; using Serilog.Events; using Serilog.Parsing; -using Serilog.Sinks.Elasticsearch; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; using Xunit; namespace Serilog.Sinks.Elasticsearch.Tests { public class CustomIndexTypeNameTests : ElasticsearchSinkTestsBase { + public CustomIndexTypeNameTests() + : base("6.0.0") + { + + } + [Fact] public void CustomIndex_And_TypeName_EndsUpInTheOutput() { diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Discrepancies/ElasticsearchSinkUniformityTestsBase.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Discrepancies/ElasticsearchSinkUniformityTestsBase.cs index 3e4ba9d6..1e3e474d 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Discrepancies/ElasticsearchSinkUniformityTestsBase.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Discrepancies/ElasticsearchSinkUniformityTestsBase.cs @@ -2,6 +2,7 @@ using System.Runtime.Serialization; using Elasticsearch.Net; using FluentAssertions; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; namespace Serilog.Sinks.Elasticsearch.Tests.Discrepancies { @@ -17,7 +18,7 @@ public void ThrowAndLogAndCatchBulkOutput(string exceptionMessage) var loggerConfig = new LoggerConfiguration() .MinimumLevel.Debug() .Enrich.WithMachineName() - .WriteTo.ColoredConsole() + .WriteTo.Console() .WriteTo.Elasticsearch(_options); var logger = loggerConfig.CreateLogger(); @@ -54,7 +55,7 @@ public void ThrowAndLogAndCatchBulkOutput(string exceptionMessage) .And.Be(exceptionMessage); var realException = firstEvent.Exceptions[0]; #if !NO_SERIALIZATION -#if !PARTIALLY_SERIALIZATION +#if NETFRAMEWORK realException.ExceptionMethod.Should().NotBeNull(); realException.ExceptionMethod.Name.Should().NotBeNullOrWhiteSpace(); realException.ExceptionMethod.AssemblyName.Should().NotBeNullOrWhiteSpace(); diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Discrepancies/NoSerializerTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Discrepancies/NoSerializerTests.cs index a5f99651..c4571d06 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Discrepancies/NoSerializerTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Discrepancies/NoSerializerTests.cs @@ -1,7 +1,4 @@ -using System; -using System.Linq; -using FluentAssertions; -using Xunit; +using Xunit; namespace Serilog.Sinks.Elasticsearch.Tests.Discrepancies { diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Domain/BulkAction.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Domain/BulkAction.cs index c754acf7..f9693bce 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Domain/BulkAction.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Domain/BulkAction.cs @@ -1,8 +1,5 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using Newtonsoft.Json; using Newtonsoft.Json.Converters; using Serilog.Events; diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/ElasticSearchLogShipperTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/ElasticSearchLogShipperTests.cs index 7de82f56..5bbe8866 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/ElasticSearchLogShipperTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/ElasticSearchLogShipperTests.cs @@ -1,10 +1,4 @@ -using Serilog.Debugging; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using Xunit; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; namespace Serilog.Sinks.Elasticsearch.Tests { diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchJsonFormatterTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchJsonFormatterTests.cs index d62a611f..f93ba0e7 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchJsonFormatterTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchJsonFormatterTests.cs @@ -4,7 +4,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Text; using Serilog.Formatting.Elasticsearch; using Xunit; diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTests.cs new file mode 100644 index 00000000..785026c4 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTests.cs @@ -0,0 +1,92 @@ +using Elasticsearch.Net; +using System.Text; +using Xunit; + +namespace Serilog.Sinks.Elasticsearch.Tests +{ + public class ElasticsearchSinkTests + { + [Theory] + [InlineData("8.0.0", "my-logevent", null)] + [InlineData("7.17.5", "my-logevent", null)] + [InlineData("6.8.1", "my-logevent", "my-logevent")] + [InlineData("8.0.0", null, null)] + [InlineData("7.17.5", null, null)] + [InlineData("6.8.1", null, "logevent")] + public void Ctor_DetectElasticsearchVersionSetToTrue_SetsTypeName(string elasticVersion, string configuredTypeName, string expectedTypeName) + { + /* ARRANGE */ + var options = new ElasticsearchSinkOptions + { + Connection = FakeResponse(elasticVersion), + TypeName = configuredTypeName + }; + + /* ACT */ + _ = ElasticsearchSinkState.Create(options); + + /* Assert */ + Assert.Equal(expectedTypeName, options.TypeName); + } + + [Theory] + [InlineData("8.0.0", "my-logevent", null)] + [InlineData("7.17.5", "my-logevent", null)] + [InlineData("6.8.1", "my-logevent", null)] + [InlineData("8.0.0", null, null)] + [InlineData("7.17.5", null, null)] + [InlineData("6.8.1", null, null)] + public void Ctor_DetectElasticsearchVersionSetToFalseAssumesVersion7_SetsTypeNameToNull(string elasticVersion, string configuredTypeName, string expectedTypeName) + { + /* ARRANGE */ + var options = new ElasticsearchSinkOptions + { + Connection = FakeResponse(elasticVersion), + DetectElasticsearchVersion = false, + TypeName = configuredTypeName + }; + + /* ACT */ + _ = ElasticsearchSinkState.Create(options); + + /* Assert */ + Assert.Equal(expectedTypeName, options.TypeName); + } + + [Theory] + [InlineData("8.0.0", "my-logevent", null)] + [InlineData("7.17.5", "my-logevent", null)] + [InlineData("6.8.1", "my-logevent", "my-logevent")] + [InlineData("8.0.0", null, null)] + [InlineData("7.17.5", null, null)] + [InlineData("6.8.1", null, "logevent")] + public void CreateLogger_DetectElasticsearchVersionSetToTrue_SetsTypeName(string elasticVersion, string configuredTypeName, string expectedTypeName) + { + /* ARRANGE */ + var options = new ElasticsearchSinkOptions + { + Connection = FakeResponse(elasticVersion), + DetectElasticsearchVersion = true, + TypeName = configuredTypeName + }; + + var loggerConfig = new LoggerConfiguration() + .MinimumLevel.Debug() + .Enrich.WithMachineName() + .WriteTo.Console() + .WriteTo.Elasticsearch(options); + + /* ACT */ + _ = loggerConfig.CreateLogger(); + + /* Assert */ + Assert.Equal(expectedTypeName, options.TypeName); + } + + private static IConnection FakeResponse(string responseText) + { + byte[] responseBody = Encoding.UTF8.GetBytes(responseText); + return new InMemoryConnection(responseBody, contentType: "text/plain; charset=UTF-8"); + } + } +} diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs b/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs deleted file mode 100644 index 6fe5c84d..00000000 --- a/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs +++ /dev/null @@ -1,196 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Elasticsearch.Net; -using FluentAssertions; -using Nest; -using Xunit; -using Serilog.Debugging; -using Serilog.Sinks.Elasticsearch.Tests.Domain; -using Nest.JsonNetSerializer; -using System.Collections; -using System.Threading; - -namespace Serilog.Sinks.Elasticsearch.Tests -{ - public abstract class ElasticsearchSinkTestsBase - { - static readonly TimeSpan TinyWait = TimeSpan.FromMilliseconds(50); - protected readonly IConnection _connection; - protected readonly ElasticsearchSinkOptions _options; - protected List _seenHttpPosts = new List(); - protected List _seenHttpHeads = new List(); - protected List> _seenHttpGets = new List>(); - protected List> _seenHttpPuts = new List>(); - private IElasticsearchSerializer _serializer; - - protected int _templateExistsReturnCode = 404; - - protected ElasticsearchSinkTestsBase() - { - _seenHttpPosts = new List(); - _seenHttpHeads = new List(); - _seenHttpGets = new List>(); - _seenHttpPuts = new List>(); - - var connectionPool = new SingleNodeConnectionPool(new Uri("http://localhost:9200")); - _connection = new ConnectionStub(_seenHttpPosts, _seenHttpHeads, _seenHttpPuts, _seenHttpGets, () => _templateExistsReturnCode); - _serializer = JsonNetSerializer.Default(LowLevelRequestResponseSerializer.Instance, new ConnectionSettings(connectionPool, _connection)); - - _options = new ElasticsearchSinkOptions(connectionPool) - { - BatchPostingLimit = 2, - //Period = TinyWait, - Connection = _connection, - Serializer = _serializer, - PipelineName = "testPipe", - }; - } - - /// - /// Returns the posted serilog messages and validates the entire bulk in the process - /// - /// - /// - protected IList GetPostedLogEvents(int expectedCount) - { - this._seenHttpPosts.Should().NotBeNullOrEmpty(); - var totalBulks = this._seenHttpPosts.SelectMany(p => p.Split(new[] { "\n" }, StringSplitOptions.RemoveEmptyEntries)).ToList(); - totalBulks.Should().NotBeNullOrEmpty().And.HaveCount(expectedCount * 2); - - var bulkActions = new List(); - for (var i = 0; i < totalBulks.Count; i += 2) - { - BulkOperation action; - try - { - action = this.Deserialize(totalBulks[i]); - } - catch (Exception e) - { - throw new Exception($"Can not deserialize into BulkOperation \r\n:{totalBulks[i]}", e); - } - action.IndexAction.Should().NotBeNull(); - action.IndexAction.Index.Should().NotBeNullOrEmpty().And.StartWith("logstash-"); - action.IndexAction.Type.Should().NotBeNullOrEmpty().And.Be("_doc"); - - SerilogElasticsearchEvent actionMetaData; - try - { - actionMetaData = this.Deserialize(totalBulks[i + 1]); - } - catch (Exception e) - { - throw new Exception( - $"Can not deserialize into SerilogElasticsearchMessage \r\n:{totalBulks[i + 1]}", e); - } - actionMetaData.Should().NotBeNull(); - bulkActions.Add(actionMetaData); - } - return bulkActions; - } - - protected T Deserialize(string json) - { - return this._serializer.Deserialize(new MemoryStream(Encoding.UTF8.GetBytes(json))); - } - - protected async Task ThrowAsync() - { - await Task.Delay(1); - throw new Exception("boom!"); - } - - protected string[] AssertSeenHttpPosts(List _seenHttpPosts, int lastN, int expectedNumberOfRequests = 2) - { - _seenHttpPosts.Should().NotBeEmpty().And.HaveCount(expectedNumberOfRequests); - var json = string.Join("", _seenHttpPosts); - var bulkJsonPieces = json.Split(new[] { '\n' }, StringSplitOptions.RemoveEmptyEntries); - - bulkJsonPieces.Count().Should().BeGreaterOrEqualTo(lastN); - var skip = Math.Max(0, bulkJsonPieces.Count() - lastN); - - return bulkJsonPieces.Skip(skip).Take(lastN).ToArray(); - } - - - public class ConnectionStub : InMemoryConnection - { - private Func _templateExistReturnCode; - private List _seenHttpHeads; - private List> _seenHttpGets; - private List _seenHttpPosts; - private List> _seenHttpPuts; - - public ConnectionStub( - List _seenHttpPosts, - List _seenHttpHeads, - List> _seenHttpPuts, - List> _seenHttpGets, - Func templateExistReturnCode - ) - { - this._seenHttpPosts = _seenHttpPosts; - this._seenHttpHeads = _seenHttpHeads; - this._seenHttpPuts = _seenHttpPuts; - this._seenHttpGets = _seenHttpGets; - this._templateExistReturnCode = templateExistReturnCode; - } - - public override TReturn Request(RequestData requestData) - { - var ms = new MemoryStream(); - if (requestData.PostData != null) - requestData.PostData.Write(ms, new ConnectionConfiguration()); - - switch (requestData.Method) - { - case HttpMethod.PUT: - _seenHttpPuts.Add(Tuple.Create(requestData.Uri, Encoding.UTF8.GetString(ms.ToArray()))); - break; - case HttpMethod.POST: - _seenHttpPosts.Add(Encoding.UTF8.GetString(ms.ToArray())); - break; - case HttpMethod.GET: - _seenHttpGets.Add(Tuple.Create(requestData.Uri, this._templateExistReturnCode())); - break; - case HttpMethod.HEAD: - _seenHttpHeads.Add(this._templateExistReturnCode()); - break; - } - - var responseStream = new MemoryStream(); - return ResponseBuilder.ToResponse(requestData, null, this._templateExistReturnCode(), Enumerable.Empty(), responseStream); - } - - public override async Task RequestAsync(RequestData requestData, CancellationToken cancellationToken) - { - var ms = new MemoryStream(); - if (requestData.PostData != null) - await requestData.PostData.WriteAsync(ms, new ConnectionConfiguration(), cancellationToken); - - switch (requestData.Method) - { - case HttpMethod.PUT: - _seenHttpPuts.Add(Tuple.Create(requestData.Uri, Encoding.UTF8.GetString(ms.ToArray()))); - break; - case HttpMethod.POST: - _seenHttpPosts.Add(Encoding.UTF8.GetString(ms.ToArray())); - break; - case HttpMethod.GET: - _seenHttpGets.Add(Tuple.Create(requestData.Uri, this._templateExistReturnCode())); - break; - case HttpMethod.HEAD: - _seenHttpHeads.Add(this._templateExistReturnCode()); - break; - } - - var responseStream = new MemoryStream(); - return await ResponseBuilder.ToResponseAsync(requestData, null, this._templateExistReturnCode(), Enumerable.Empty(), responseStream, null, cancellationToken); - } - } - } -} \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/ExceptionAsJsonObjectFormatterTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/ExceptionAsJsonObjectFormatterTests.cs index aa1b76f7..92a2322f 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/ExceptionAsJsonObjectFormatterTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/ExceptionAsJsonObjectFormatterTests.cs @@ -9,6 +9,7 @@ using Serilog.Formatting.Elasticsearch; using Serilog.Parsing; using Serilog.Sinks.Elasticsearch.Tests.Domain; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; namespace Serilog.Sinks.Elasticsearch.Tests { @@ -51,7 +52,7 @@ public void WhenLogging_WithException_ExceptionShouldBeRenderedInExceptionField( var exceptionInfo = eventWritten.Exception; exceptionInfo.Should().NotBeNull(); exceptionInfo.Message.Should().Be(expectedExceptionMessage); -#if !DOTNETCORE +#if NETFRAMEWORK exceptionInfo.ClassName.Should().Be("System.Exception"); #endif } diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/IndexDeciderTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/IndexDeciderTests.cs index 77fbd894..18439d9d 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/IndexDeciderTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/IndexDeciderTests.cs @@ -1,13 +1,13 @@ using System; using System.Collections.Generic; -using System.Linq; using FluentAssertions; using Xunit; using Serilog.Events; using Serilog.Parsing; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; namespace Serilog.Sinks.Elasticsearch.Tests -{ +{ public class IndexDeciderTests : ElasticsearchSinkTestsBase { [Fact] diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/InlineFieldsTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/InlineFieldsTests.cs index c409ec51..c9ccb8ef 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/InlineFieldsTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/InlineFieldsTests.cs @@ -1,10 +1,10 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Threading.Tasks; using FluentAssertions; using Serilog.Events; using Serilog.Parsing; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; using Xunit; namespace Serilog.Sinks.Elasticsearch.Tests diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Properties/AssemblyInfo.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Properties/AssemblyInfo.cs index 7f8e8012..a40a0fb5 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Properties/AssemblyInfo.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Properties/AssemblyInfo.cs @@ -1,5 +1,4 @@ using System.Reflection; -using System.Runtime.CompilerServices; using System.Runtime.InteropServices; // General Information about an assembly is controlled through the following diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/PropertyNameTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/PropertyNameTests.cs index ed6d8832..cb7217ed 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/PropertyNameTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/PropertyNameTests.cs @@ -1,11 +1,11 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Threading.Tasks; using FluentAssertions; using Xunit; using Serilog.Events; using Serilog.Parsing; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; namespace Serilog.Sinks.Elasticsearch.Tests { diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/RealExceptionNoSerializerTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/RealExceptionNoSerializerTests.cs index 9c69af3e..8143a293 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/RealExceptionNoSerializerTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/RealExceptionNoSerializerTests.cs @@ -1,11 +1,11 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Threading.Tasks; using FluentAssertions; using Xunit; using Serilog.Events; using Serilog.Parsing; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; namespace Serilog.Sinks.Elasticsearch.Tests { diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/RealExceptionTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/RealExceptionTests.cs index 0f6caf23..1063e882 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/RealExceptionTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/RealExceptionTests.cs @@ -1,12 +1,11 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Threading.Tasks; using FluentAssertions; -using Nest; using Xunit; using Serilog.Events; using Serilog.Parsing; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; namespace Serilog.Sinks.Elasticsearch.Tests { diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj b/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj index f9755252..2fd5ecdf 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj @@ -1,13 +1,14 @@  - netcoreapp3.1 + net6.0;net7.0;net48 Serilog.Sinks.Elasticsearch.Tests Serilog.Sinks.Elasticsearch.Tests True latest True + false true false @@ -27,27 +28,28 @@ - - - - + + + + + Serilog.snk - - - + + + + + - - @@ -56,27 +58,29 @@ - - - - - + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - + + + - - - + + + - + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + @@ -104,14 +108,6 @@ - - - - - - $(DefineConstants);DOTNETCORE;PARTIALLY_SERIALIZATION - - diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Stubs/ConnectionStub.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Stubs/ConnectionStub.cs new file mode 100644 index 00000000..c1ae51e9 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Stubs/ConnectionStub.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading.Tasks; +using Elasticsearch.Net; +using System.Threading; + +namespace Serilog.Sinks.Elasticsearch.Tests.Stubs +{ + internal class ConnectionStub : InMemoryConnection + { + private readonly Func _templateExistReturnCode; + private readonly List _seenHttpHeads; + private readonly List> _seenHttpGets; + private readonly List _seenHttpPosts; + private readonly List> _seenHttpPuts; + + private readonly string _productVersion; + + /// + /// Elasticsearch.NET client version 7.16 or higher + /// uses pre-flight request, before any other request is served, + /// to check product (Elasticsearch) and version of the product. + /// It can be seen on property. + /// + private bool _productCheckDone; + + public ConnectionStub( + List _seenHttpPosts, + List _seenHttpHeads, + List> _seenHttpPuts, + List> _seenHttpGets, + Func templateExistReturnCode, + string productVersion = "8.6.0" + ) : base() + { + this._seenHttpPosts = _seenHttpPosts; + this._seenHttpHeads = _seenHttpHeads; + this._seenHttpPuts = _seenHttpPuts; + this._seenHttpGets = _seenHttpGets; + _templateExistReturnCode = templateExistReturnCode; + _productVersion = productVersion; + } + + public override TReturn Request(RequestData requestData) + { + if (_productCheckDone == false) + { + if (requestData.Method != HttpMethod.GET || requestData.PathAndQuery != string.Empty) + throw new InvalidOperationException( + $"{nameof(ConnectionStub)} expects first request" + + $" to be productCheck pre-flight request"); + + _productCheckDone = true; + return ReturnConnectionStatus(requestData); // root page returned + } + + byte[] responseBytes = Array.Empty(); + if (requestData.PostData != null) + { + using var ms = new MemoryStream(); + requestData.PostData.Write(ms, new ConnectionConfiguration()); + responseBytes = ms.ToArray(); + } + + int responseStatusCode = 200; + string contentType = null; + + switch (requestData.Method) + { + case HttpMethod.PUT: + _seenHttpPuts.Add(Tuple.Create(requestData.Uri, Encoding.UTF8.GetString(responseBytes))); + break; + case HttpMethod.POST: + _seenHttpPosts.Add(Encoding.UTF8.GetString(responseBytes)); + break; + case HttpMethod.GET: + switch (requestData.Uri.PathAndQuery.ToLower()) + { + case "/": + // ReturnConnectionStatus(...) call at the bottom will return dummy product page + // when root "/" is requested. + break; + case "/_cat/nodes": + case "/_cat/nodes?h=v": + responseBytes = Encoding.UTF8.GetBytes(_productVersion); + responseStatusCode = 200; + contentType = "text/plain; charset=UTF-8"; + break; + } + _seenHttpGets.Add(Tuple.Create(requestData.Uri, responseStatusCode)); + break; + case HttpMethod.HEAD: + if (requestData.Uri.PathAndQuery.ToLower().StartsWith("/_template/")) + { + responseStatusCode = _templateExistReturnCode(); + } + _seenHttpHeads.Add(responseStatusCode); + break; + } + + return ReturnConnectionStatus(requestData, responseBytes, responseStatusCode, contentType); + } + + public override Task RequestAsync(RequestData requestData, CancellationToken cancellationToken) + { + return Task.FromResult(Request(requestData)); + } + } +} \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Stubs/ElasticsearchSinkTestsBase.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Stubs/ElasticsearchSinkTestsBase.cs new file mode 100644 index 00000000..e5b0717a --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Stubs/ElasticsearchSinkTestsBase.cs @@ -0,0 +1,137 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Elasticsearch.Net; +using FluentAssertions; +using Nest; +using Serilog.Sinks.Elasticsearch.Tests.Domain; +using Nest.JsonNetSerializer; +using Newtonsoft.Json.Linq; + +namespace Serilog.Sinks.Elasticsearch.Tests.Stubs +{ + public abstract partial class ElasticsearchSinkTestsBase + { + static readonly TimeSpan TinyWait = TimeSpan.FromMilliseconds(50); + protected readonly IConnection _connection; + protected readonly ElasticsearchSinkOptions _options; + protected List _seenHttpPosts = new List(); + protected List _seenHttpHeads = new List(); + protected List> _seenHttpGets = new List>(); + protected List> _seenHttpPuts = new List>(); + private IElasticsearchSerializer _serializer; + + protected int _templateExistsReturnCode = 404; + + protected ElasticsearchSinkTestsBase(string productVersion = "8.6.0") + { + _seenHttpPosts = new List(); + _seenHttpHeads = new List(); + _seenHttpGets = new List>(); + _seenHttpPuts = new List>(); + + var connectionPool = new SingleNodeConnectionPool(new Uri("http://localhost:9200")); + _connection = new ConnectionStub(_seenHttpPosts, + _seenHttpHeads, + _seenHttpPuts, + _seenHttpGets, + () => _templateExistsReturnCode, + productVersion); + _serializer = JsonNetSerializer.Default(LowLevelRequestResponseSerializer.Instance, new ConnectionSettings(connectionPool, _connection)); + + _options = new ElasticsearchSinkOptions(connectionPool) + { + BatchPostingLimit = 2, + //Period = TinyWait, + Connection = _connection, + Serializer = _serializer, + PipelineName = "testPipe", + }; + } + + /// + /// Returns the posted serilog messages and validates the entire bulk in the process + /// + /// + /// + protected IList GetPostedLogEvents(int expectedCount) + { + _seenHttpPosts.Should().NotBeNullOrEmpty(); + var totalBulks = _seenHttpPosts.SelectMany(p => p.Split(new[] { "\n" }, StringSplitOptions.RemoveEmptyEntries)).ToList(); + totalBulks.Should().NotBeNullOrEmpty().And.HaveCount(expectedCount * 2); + + var bulkActions = new List(); + for (var i = 0; i < totalBulks.Count; i += 2) + { + BulkOperation action; + try + { + action = Deserialize(totalBulks[i]); + } + catch (Exception e) + { + throw new Exception($"Can not deserialize into BulkOperation \r\n:{totalBulks[i]}", e); + } + action.IndexAction.Should().NotBeNull(); + action.IndexAction.Index.Should().NotBeNullOrEmpty().And.StartWith("logstash-"); + action.IndexAction.Type.Should().BeNull(); + + SerilogElasticsearchEvent actionMetaData; + try + { + actionMetaData = Deserialize(totalBulks[i + 1]); + } + catch (Exception e) + { + throw new Exception( + $"Can not deserialize into SerilogElasticsearchMessage \r\n:{totalBulks[i + 1]}", e); + } + actionMetaData.Should().NotBeNull(); + bulkActions.Add(actionMetaData); + } + return bulkActions; + } + + protected T Deserialize(string json) + { + return _serializer.Deserialize(new MemoryStream(Encoding.UTF8.GetBytes(json))); + } + + protected async Task ThrowAsync() + { + await Task.Delay(1); + throw new Exception("boom!"); + } + + protected string[] AssertSeenHttpPosts(List _seenHttpPosts, int lastN, int expectedNumberOfRequests = 2) + { + _seenHttpPosts.Should().NotBeEmpty().And.HaveCount(expectedNumberOfRequests); + var json = string.Join("", _seenHttpPosts); + var bulkJsonPieces = json.Split(new[] { '\n' }, StringSplitOptions.RemoveEmptyEntries); + + bulkJsonPieces.Count().Should().BeGreaterOrEqualTo(lastN); + var skip = Math.Max(0, bulkJsonPieces.Count() - lastN); + + return bulkJsonPieces.Skip(skip).Take(lastN).ToArray(); + } + + protected void JsonEquals(string json, string embeddedResourceNameEndsWith) + { +#if NETFRAMEWORK + var assembly = System.Reflection.Assembly.GetExecutingAssembly(); +#else + var assembly = GetType().Assembly; +#endif + var expected = TestDataHelper.ReadEmbeddedResource(assembly, embeddedResourceNameEndsWith); + + var nJson = JObject.Parse(json); + var nOtherJson = JObject.Parse(expected); + var equals = JToken.DeepEquals(nJson, nOtherJson); + if (equals) return; + expected.Should().BeEquivalentTo(json); + } + } +} \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/DiscoverVersionHandlesUnavailableServerTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/DiscoverVersionHandlesUnavailableServerTests.cs index b80446f4..90ff885a 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/DiscoverVersionHandlesUnavailableServerTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/DiscoverVersionHandlesUnavailableServerTests.cs @@ -4,6 +4,7 @@ using FluentAssertions; using Xunit; using Serilog.Debugging; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; namespace Serilog.Sinks.Elasticsearch.Tests.Templating { diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/DiscoverVersionTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/DiscoverVersionTests.cs index 7d767744..d922a3e7 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/DiscoverVersionTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/DiscoverVersionTests.cs @@ -1,5 +1,6 @@ using System; using FluentAssertions; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; using Xunit; namespace Serilog.Sinks.Elasticsearch.Tests.Templating @@ -15,7 +16,7 @@ public DiscoverVersionTests() var loggerConfig = new LoggerConfiguration() .MinimumLevel.Debug() .Enrich.WithMachineName() - .WriteTo.ColoredConsole() + .WriteTo.Console() .WriteTo.Elasticsearch(_options); var logger = loggerConfig.CreateLogger(); diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/DoNotRegisterTemplateIfItExists.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/DoNotRegisterTemplateIfItExists.cs index 96300ab6..4cef9730 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/DoNotRegisterTemplateIfItExists.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/DoNotRegisterTemplateIfItExists.cs @@ -1,5 +1,6 @@ using System; using FluentAssertions; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; using Xunit; namespace Serilog.Sinks.Elasticsearch.Tests.Templating @@ -14,7 +15,7 @@ private void DoRegister() var loggerConfig = new LoggerConfiguration() .MinimumLevel.Debug() .Enrich.WithMachineName() - .WriteTo.ColoredConsole() + .WriteTo.Console() .WriteTo.Elasticsearch(_options); var logger = loggerConfig.CreateLogger(); diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/OverwriteTemplateTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/OverwriteTemplateTests.cs index a37936f6..aa63d1a5 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/OverwriteTemplateTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/OverwriteTemplateTests.cs @@ -1,16 +1,13 @@ using System; -using System.IO; -using System.Reflection; using FluentAssertions; -using Newtonsoft.Json.Linq; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; using Xunit; namespace Serilog.Sinks.Elasticsearch.Tests.Templating { public class OverwriteTemplateTests : ElasticsearchSinkTestsBase { - - public void DoRegister() + private void DoRegister() { _templateExistsReturnCode = 200; @@ -19,7 +16,7 @@ public void DoRegister() var loggerConfig = new LoggerConfiguration() .MinimumLevel.Debug() .Enrich.WithMachineName() - .WriteTo.ColoredConsole() + .WriteTo.Console() .WriteTo.Elasticsearch(_options); var logger = loggerConfig.CreateLogger(); diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/RegisterCustomTemplateTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/RegisterCustomTemplateTests.cs index 40e59d3e..3379f113 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/RegisterCustomTemplateTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/RegisterCustomTemplateTests.cs @@ -1,8 +1,6 @@ using System; -using System.IO; -using System.Reflection; using FluentAssertions; -using Newtonsoft.Json.Linq; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; using Xunit; namespace Serilog.Sinks.Elasticsearch.Tests.Templating @@ -19,7 +17,7 @@ public RegisterCustomTemplateTests() var loggerConfig = new LoggerConfiguration() .MinimumLevel.Debug() .Enrich.WithMachineName() - .WriteTo.ColoredConsole() + .WriteTo.Console() .WriteTo.Elasticsearch(_options); var logger = loggerConfig.CreateLogger(); diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SendsTemplateHandlesUnavailableServerTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SendsTemplateHandlesUnavailableServerTests.cs index 2f1b0a51..cf85bea1 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SendsTemplateHandlesUnavailableServerTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SendsTemplateHandlesUnavailableServerTests.cs @@ -4,6 +4,7 @@ using FluentAssertions; using Xunit; using Serilog.Debugging; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; namespace Serilog.Sinks.Elasticsearch.Tests.Templating { diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SendsTemplateTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SendsTemplateTests.cs index 161d0bac..9e86c119 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SendsTemplateTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SendsTemplateTests.cs @@ -1,7 +1,6 @@ using System; -using System.Reflection; using FluentAssertions; -using Newtonsoft.Json.Linq; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; using Xunit; namespace Serilog.Sinks.Elasticsearch.Tests.Templating @@ -12,12 +11,13 @@ public class SendsTemplateTests : ElasticsearchSinkTestsBase public SendsTemplateTests() { + _options.DetectElasticsearchVersion = false; _options.AutoRegisterTemplate = true; var loggerConfig = new LoggerConfiguration() .MinimumLevel.Debug() .Enrich.WithMachineName() - .WriteTo.ColoredConsole() + .WriteTo.Console() .WriteTo.Elasticsearch(_options); var logger = loggerConfig.CreateLogger(); @@ -32,10 +32,9 @@ public SendsTemplateTests() } [Fact] - public void ShouldRegisterTheCorrectTemplateOnRegistration() + public void ShouldRegisterTheVersion7TemplateOnRegistrationWhenDetectElasticsearchVersionFalse() { - var method = typeof(SendsTemplateTests).GetMethod(nameof(ShouldRegisterTheCorrectTemplateOnRegistration)); - JsonEquals(_templatePut.Item2, method, "template"); + JsonEquals(_templatePut.Item2, "template_v7_no-aliases.json"); } [Fact] @@ -44,21 +43,5 @@ public void TemplatePutToCorrectUrl() var uri = _templatePut.Item1; uri.AbsolutePath.Should().Be("/_template/serilog-events-template"); } - - protected void JsonEquals(string json, MethodBase method, string fileName = null) - { -#if DOTNETCORE - var assembly = typeof(SendsTemplateTests).GetTypeInfo().Assembly; -#else - var assembly = Assembly.GetExecutingAssembly(); -#endif - var expected = TestDataHelper.ReadEmbeddedResource(assembly, "template.json"); - - var nJson = JObject.Parse(json); - var nOtherJson = JObject.Parse(expected); - var equals = JToken.DeepEquals(nJson, nOtherJson); - if (equals) return; - expected.Should().BeEquivalentTo(json); - } } } \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv5TemplateTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv5TemplateTests.cs index 625dcc9d..dae86be3 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv5TemplateTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv5TemplateTests.cs @@ -1,7 +1,6 @@ using System; -using System.Reflection; using FluentAssertions; -using Newtonsoft.Json.Linq; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; using Xunit; namespace Serilog.Sinks.Elasticsearch.Tests.Templating @@ -12,13 +11,14 @@ public class Sendsv5TemplateTests : ElasticsearchSinkTestsBase public Sendsv5TemplateTests() { + _options.DetectElasticsearchVersion = false; _options.AutoRegisterTemplate = true; _options.AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv5; var loggerConfig = new LoggerConfiguration() .MinimumLevel.Debug() .Enrich.WithMachineName() - .WriteTo.ColoredConsole() + .WriteTo.Console() .WriteTo.Elasticsearch(_options); var logger = loggerConfig.CreateLogger(); @@ -33,11 +33,9 @@ public Sendsv5TemplateTests() } [Fact] - public void ShouldRegisterTheCorrectTemplateOnRegistration() + public void ShouldRegisterTheVersion7TemplateOnRegistrationWhenDetectElasticsearchVersionFalseAndAutoRegisterTemplateVersionIsESv5() { - - var method = typeof(Sendsv5TemplateTests).GetMethod(nameof(ShouldRegisterTheCorrectTemplateOnRegistration)); - JsonEquals(_templatePut.Item2, method, "template_v5.json"); + JsonEquals(_templatePut.Item2, "template_v5.json"); } [Fact] @@ -46,21 +44,5 @@ public void TemplatePutToCorrectUrl() var uri = _templatePut.Item1; uri.AbsolutePath.Should().Be("/_template/serilog-events-template"); } - - protected void JsonEquals(string json, MethodBase method, string fileName = null) - { -#if DOTNETCORE - var assembly = typeof(Sendsv5TemplateTests).GetTypeInfo().Assembly; -#else - var assembly = Assembly.GetExecutingAssembly(); -#endif - var expected = TestDataHelper.ReadEmbeddedResource(assembly, fileName ?? "template.json"); - - var nJson = JObject.Parse(json); - var nOtherJson = JObject.Parse(expected); - var equals = JToken.DeepEquals(nJson, nOtherJson); - if (equals) return; - expected.Should().BeEquivalentTo(json); - } } } \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv6TemplateTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv6TemplateTests.cs index 8764aae2..fc3018ac 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv6TemplateTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv6TemplateTests.cs @@ -1,7 +1,6 @@ using System; -using System.Reflection; using FluentAssertions; -using Newtonsoft.Json.Linq; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; using Xunit; namespace Serilog.Sinks.Elasticsearch.Tests.Templating @@ -11,6 +10,7 @@ public class Sendsv6TemplateTests : ElasticsearchSinkTestsBase private readonly Tuple _templatePut; public Sendsv6TemplateTests() + : base("6.0.0") { _options.AutoRegisterTemplate = true; _options.AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv6; @@ -18,7 +18,7 @@ public Sendsv6TemplateTests() var loggerConfig = new LoggerConfiguration() .MinimumLevel.Debug() .Enrich.WithMachineName() - .WriteTo.ColoredConsole() + .WriteTo.Console() .WriteTo.Elasticsearch(_options); var logger = loggerConfig.CreateLogger(); @@ -33,11 +33,9 @@ public Sendsv6TemplateTests() } [Fact] - public void ShouldRegisterTheCorrectTemplateOnRegistration() + public void ShouldRegisterTheVersion6TemplateOnRegistrationWhenDetectedElasticsearchVersionIsV6() { - - var method = typeof(Sendsv6TemplateTests).GetMethod(nameof(ShouldRegisterTheCorrectTemplateOnRegistration)); - JsonEquals(_templatePut.Item2, method, "template_v6.json"); + JsonEquals(_templatePut.Item2, "template_v6.json"); } [Fact] @@ -46,21 +44,5 @@ public void TemplatePutToCorrectUrl() var uri = _templatePut.Item1; uri.AbsolutePath.Should().Be("/_template/serilog-events-template"); } - - protected void JsonEquals(string json, MethodBase method, string fileName = null) - { -#if DOTNETCORE - var assembly = typeof(Sendsv6TemplateTests).GetTypeInfo().Assembly; -#else - var assembly = Assembly.GetExecutingAssembly(); -#endif - var expected = TestDataHelper.ReadEmbeddedResource(assembly, fileName ?? "template.json"); - - var nJson = JObject.Parse(json); - var nOtherJson = JObject.Parse(expected); - var equals = JToken.DeepEquals(nJson, nOtherJson); - if (equals) return; - expected.Should().BeEquivalentTo(json); - } } } \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv7TemplateTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv7TemplateTests.cs index 9413d1ed..e8d695f6 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv7TemplateTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv7TemplateTests.cs @@ -1,7 +1,6 @@ using System; -using System.Reflection; using FluentAssertions; -using Newtonsoft.Json.Linq; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; using Xunit; namespace Serilog.Sinks.Elasticsearch.Tests.Templating @@ -11,6 +10,7 @@ public class Sendsv7TemplateTests : ElasticsearchSinkTestsBase private readonly Tuple _templatePut; public Sendsv7TemplateTests() + : base("7.0.0") { _options.AutoRegisterTemplate = true; _options.AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7; @@ -19,7 +19,7 @@ public Sendsv7TemplateTests() var loggerConfig = new LoggerConfiguration() .MinimumLevel.Debug() .Enrich.WithMachineName() - .WriteTo.ColoredConsole() + .WriteTo.Console() .WriteTo.Elasticsearch(_options); var logger = loggerConfig.CreateLogger(); @@ -34,11 +34,9 @@ public Sendsv7TemplateTests() } [Fact] - public void ShouldRegisterTheCorrectTemplateOnRegistration() + public void ShouldRegisterTheVersion7TemplateOnRegistrationWhenDetectedElasticsearchVersionIsV7() { - - var method = typeof(Sendsv7TemplateTests).GetMethod(nameof(ShouldRegisterTheCorrectTemplateOnRegistration)); - JsonEquals(_templatePut.Item2, method, "template_v7.json"); + JsonEquals(_templatePut.Item2, "template_v7.json"); } [Fact] @@ -47,21 +45,5 @@ public void TemplatePutToCorrectUrl() var uri = _templatePut.Item1; uri.AbsolutePath.Should().Be("/_template/serilog-events-template"); } - - protected void JsonEquals(string json, MethodBase method, string fileName = null) - { -#if DOTNETCORE - var assembly = typeof(Sendsv7TemplateTests).GetTypeInfo().Assembly; -#else - var assembly = Assembly.GetExecutingAssembly(); -#endif - var expected = TestDataHelper.ReadEmbeddedResource(assembly, fileName ?? "template.json"); - - var nJson = JObject.Parse(json); - var nOtherJson = JObject.Parse(expected); - var equals = JToken.DeepEquals(nJson, nOtherJson); - if (equals) return; - expected.Should().BeEquivalentTo(json); - } } } \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv8TemplateTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv8TemplateTests.cs new file mode 100644 index 00000000..22de419f --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/Sendsv8TemplateTests.cs @@ -0,0 +1,48 @@ +using System; +using FluentAssertions; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; +using Xunit; + +namespace Serilog.Sinks.Elasticsearch.Tests.Templating +{ + public class Sendsv8TemplateTests : ElasticsearchSinkTestsBase + { + private readonly Tuple _templatePut; + + public Sendsv8TemplateTests() + { + _options.AutoRegisterTemplate = true; + _options.AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv8; + _options.IndexAliases = new string[] { "logstash" }; + + var loggerConfig = new LoggerConfiguration() + .MinimumLevel.Debug() + .Enrich.WithMachineName() + .WriteTo.Console() + .WriteTo.Elasticsearch(_options); + + var logger = loggerConfig.CreateLogger(); + using (logger as IDisposable) + { + logger.Error("Test exception. Should not contain an embedded exception object."); + } + + this._seenHttpPosts.Should().NotBeNullOrEmpty().And.HaveCount(1); + this._seenHttpPuts.Should().NotBeNullOrEmpty().And.HaveCount(1); + _templatePut = this._seenHttpPuts[0]; + } + + [Fact] + public void ShouldRegisterTheVersion6TemplateOnRegistrationWhenDetectedElasticsearchVersionIsV8() + { + JsonEquals(_templatePut.Item2, "template_v8.json"); + } + + [Fact] + public void TemplatePutToCorrectUrl() + { + var uri = _templatePut.Item1; + uri.AbsolutePath.Should().Be("/_index_template/serilog-events-template"); + } + } +} \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetElasticsearchSinkOptions.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetElasticsearchSinkOptions.cs index ba4e24da..f48f5638 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetElasticsearchSinkOptions.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetElasticsearchSinkOptions.cs @@ -1,5 +1,6 @@ using System; using FluentAssertions; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; using Xunit; namespace Serilog.Sinks.Elasticsearch.Tests.Templating diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetFiveReplicasInTemplateTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetFiveReplicasInTemplateTests.cs index 602e3377..29654081 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetFiveReplicasInTemplateTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetFiveReplicasInTemplateTests.cs @@ -1,7 +1,6 @@ using System; -using System.Reflection; using FluentAssertions; -using Newtonsoft.Json.Linq; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; using Xunit; namespace Serilog.Sinks.Elasticsearch.Tests.Templating @@ -18,7 +17,7 @@ public SetFiveReplicasInTemplateTests() var loggerConfig = new LoggerConfiguration() .MinimumLevel.Debug() .Enrich.WithMachineName() - .WriteTo.ColoredConsole() + .WriteTo.Console() .WriteTo.Elasticsearch(_options); var logger = loggerConfig.CreateLogger(); @@ -35,31 +34,14 @@ public SetFiveReplicasInTemplateTests() [Fact] public void ShouldRegisterTheCorrectTemplateOnRegistration() { - var method = typeof(SendsTemplateTests).GetMethod(nameof(ShouldRegisterTheCorrectTemplateOnRegistration)); - JsonEquals(_templatePut.Item2, method, "template"); + JsonEquals(_templatePut.Item2, "template_v8_no-aliases_5replicas.json"); } [Fact] public void TemplatePutToCorrectUrl() { var uri = _templatePut.Item1; - uri.AbsolutePath.Should().Be("/_template/serilog-events-template"); - } - - protected void JsonEquals(string json, MethodBase method, string fileName = null) - { -#if DOTNETCORE - var assembly = typeof(SendsTemplateTests).GetTypeInfo().Assembly; -#else - var assembly = Assembly.GetExecutingAssembly(); -#endif - var expected = TestDataHelper.ReadEmbeddedResource(assembly, "template_5replicas.json"); - - var nJson = JObject.Parse(json); - var nOtherJson = JObject.Parse(expected); - var equals = JToken.DeepEquals(nJson, nOtherJson); - if (equals) return; - expected.Should().BeEquivalentTo(json); + uri.AbsolutePath.Should().Be("/_index_template/serilog-events-template"); } } } \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetTwoShardsInTemplateTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetTwoShardsInTemplateTests.cs index 4a06c4d6..0543365c 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetTwoShardsInTemplateTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetTwoShardsInTemplateTests.cs @@ -1,7 +1,6 @@ using System; -using System.Reflection; using FluentAssertions; -using Newtonsoft.Json.Linq; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; using Xunit; namespace Serilog.Sinks.Elasticsearch.Tests.Templating @@ -19,7 +18,7 @@ public SetTwoShardsInTemplateTests() var loggerConfig = new LoggerConfiguration() .MinimumLevel.Debug() .Enrich.WithMachineName() - .WriteTo.ColoredConsole() + .WriteTo.Console() .WriteTo.Elasticsearch(_options); var logger = loggerConfig.CreateLogger(); @@ -36,31 +35,14 @@ public SetTwoShardsInTemplateTests() [Fact] public void ShouldRegisterTheCorrectTemplateOnRegistration() { - var method = typeof(SendsTemplateTests).GetMethod(nameof(ShouldRegisterTheCorrectTemplateOnRegistration)); - JsonEquals(_templatePut.Item2, method, "template"); + JsonEquals(_templatePut.Item2, "template_v8_no-aliases_2shards.json"); } [Fact] public void TemplatePutToCorrectUrl() { var uri = _templatePut.Item1; - uri.AbsolutePath.Should().Be("/_template/serilog-events-template"); - } - - protected void JsonEquals(string json, MethodBase method, string fileName = null) - { -#if DOTNETCORE - var assembly = typeof(SendsTemplateTests).GetTypeInfo().Assembly; -#else - var assembly = Assembly.GetExecutingAssembly(); -#endif - var expected = TestDataHelper.ReadEmbeddedResource(assembly, "template_2shards.json"); - - var nJson = JObject.Parse(json); - var nOtherJson = JObject.Parse(expected); - var equals = JToken.DeepEquals(nJson, nOtherJson); - if (equals) return; - expected.Should().BeEquivalentTo(json); + uri.AbsolutePath.Should().Be("/_index_template/serilog-events-template"); } } } \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetZeroReplicasInTemplateTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetZeroReplicasInTemplateTests.cs index 47c87812..60b7451c 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetZeroReplicasInTemplateTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SetZeroReplicasInTemplateTests.cs @@ -1,7 +1,6 @@ using System; -using System.Reflection; using FluentAssertions; -using Newtonsoft.Json.Linq; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; using Xunit; namespace Serilog.Sinks.Elasticsearch.Tests.Templating @@ -18,7 +17,7 @@ public SetZeroReplicasInTemplateTests() var loggerConfig = new LoggerConfiguration() .MinimumLevel.Debug() .Enrich.WithMachineName() - .WriteTo.ColoredConsole() + .WriteTo.Console() .WriteTo.Elasticsearch(_options); var logger = loggerConfig.CreateLogger(); @@ -35,31 +34,14 @@ public SetZeroReplicasInTemplateTests() [Fact] public void ShouldRegisterTheCorrectTemplateOnRegistration() { - var method = typeof(SendsTemplateTests).GetMethod(nameof(ShouldRegisterTheCorrectTemplateOnRegistration)); - JsonEquals(_templatePut.Item2, method, "template"); + JsonEquals(_templatePut.Item2, "template_v8_no-aliases_0replicas.json"); } [Fact] public void TemplatePutToCorrectUrl() { var uri = _templatePut.Item1; - uri.AbsolutePath.Should().Be("/_template/serilog-events-template"); - } - - protected void JsonEquals(string json, MethodBase method, string fileName = null) - { -#if DOTNETCORE - var assembly = typeof(SendsTemplateTests).GetTypeInfo().Assembly; -#else - var assembly = Assembly.GetExecutingAssembly(); -#endif - var expected = TestDataHelper.ReadEmbeddedResource(assembly, "template_0replicas.json"); - - var nJson = JObject.Parse(json); - var nOtherJson = JObject.Parse(expected); - var equals = JToken.DeepEquals(nJson, nOtherJson); - if (equals) return; - expected.Should().BeEquivalentTo(json); + uri.AbsolutePath.Should().Be("/_index_template/serilog-events-template"); } } } \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/TemplateMatchTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/TemplateMatchTests.cs index 16b549a2..2fe1fcc6 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/TemplateMatchTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/TemplateMatchTests.cs @@ -1,8 +1,6 @@ using System; -using System.IO; -using System.Reflection; using FluentAssertions; -using Newtonsoft.Json.Linq; +using Serilog.Sinks.Elasticsearch.Tests.Stubs; using Xunit; namespace Serilog.Sinks.Elasticsearch.Tests.Templating @@ -12,6 +10,7 @@ public class TemplateMatchTests : ElasticsearchSinkTestsBase private readonly Tuple _templatePut; public TemplateMatchTests() + : base("7.0.0") { _options.AutoRegisterTemplate = true; _options.IndexFormat = "dailyindex-{0:yyyy.MM.dd}-mycompany"; @@ -19,7 +18,7 @@ public TemplateMatchTests() var loggerConfig = new LoggerConfiguration() .MinimumLevel.Debug() .Enrich.WithMachineName() - .WriteTo.ColoredConsole() + .WriteTo.Console() .WriteTo.Elasticsearch(_options); var logger = loggerConfig.CreateLogger(); @@ -46,7 +45,7 @@ public void TemplatePutToCorrectUrl() public void TemplateMatchShouldReflectConfiguredIndexFormat() { var json = this._templatePut.Item2; - json.Should().Contain(@"""template"":""dailyindex-*-mycompany"""); + json.Should().Contain(@"""index_patterns"":[""dailyindex-*-mycompany""]"); } } diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_2shards.json b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_2shards.json deleted file mode 100644 index 82a097f1..00000000 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_2shards.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "template": "logstash-*", - "settings": { - "index.refresh_interval": "5s", - "number_of_shards": "2", - "number_of_replicas": "0" - }, - "mappings": { - "_default_": { - "_all": { - "enabled": true, - "omit_norms" : true - }, - "dynamic_templates": [ - { - "numerics_in_fields": { - "path_match":"fields\\.[\\d+]$", - "match_pattern":"regex", - "mapping": { - "type":"string", - "index":"analyzed", - "omit_norms":true - } - } - }, - { - "string_fields": { - "match": "*", - "match_mapping_type": "string", - "mapping": { - "type": "string", - "index": "analyzed", - "omit_norms": true, - "fields": { - "raw": { - "type": "string", - "index": "not_analyzed", - "ignore_above": 256 - } - } - } - } - } - ], - "properties": { - "message": { - "type": "string", - "index": "analyzed" - }, - "exceptions": { - "type": "nested", - "properties": { - "Depth": { - "type": "integer" - }, - "RemoteStackIndex": { - "type": "integer" - }, - "HResult": { - "type": "integer" - }, - "StackTraceString": { - "type": "string", - "index": "analyzed" - }, - "RemoteStackTraceString": { - "type": "string", - "index": "analyzed" - }, - "ExceptionMessage": { - "type": "object", - "properties": { - "MemberType": { - "type": "integer" - } - } - } - } - } - } - } - } -} \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v7_no-aliases.json b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v7_no-aliases.json new file mode 100644 index 00000000..07bbb617 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v7_no-aliases.json @@ -0,0 +1,77 @@ +{ + "index_patterns": [ "logstash-*" ], + "settings": { + "index.refresh_interval": "5s" + }, + "mappings": { + "dynamic_templates": [ + { + "numerics_in_fields": { + "path_match": "fields\\.[\\d+]$", + "match_pattern": "regex", + "mapping": { + "type": "text", + "index": true, + "norms": false + } + } + }, + { + "string_fields": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "text", + "index": true, + "norms": false, + "fields": { + "raw": { + "type": "keyword", + "index": true, + "ignore_above": 256 + } + } + } + } + } + ], + "properties": { + "message": { + "type": "text", + "index": true + }, + "exceptions": { + "type": "nested", + "properties": { + "Depth": { + "type": "integer" + }, + "RemoteStackIndex": { + "type": "integer" + }, + "HResult": { + "type": "integer" + }, + "StackTraceString": { + "type": "text", + "index": true + }, + "RemoteStackTraceString": { + "type": "text", + "index": true + }, + "ExceptionMessage": { + "type": "object", + "properties": { + "MemberType": { + "type": "integer" + } + } + } + } + } + } + }, + "aliases": { + } +} diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template.json b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v8.json similarity index 59% rename from test/Serilog.Sinks.Elasticsearch.Tests/Templating/template.json rename to test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v8.json index 6a45a045..af3f5b3e 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template.json +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v8.json @@ -1,38 +1,34 @@ { - "template": "logstash-*", - "settings": { - "index.refresh_interval": "5s" - }, - "mappings": { - "_default_": { - "_all": { - "enabled": true, - "omit_norms" : true - }, + "index_patterns": [ "logstash-*" ], + "template": { + "settings": { + "index.refresh_interval": "5s" + }, + "mappings": { "dynamic_templates": [ { - "numerics_in_fields": { - "path_match":"fields\\.[\\d+]$", - "match_pattern":"regex", - "mapping": { - "type":"string", - "index":"analyzed", - "omit_norms":true - } - } + "numerics_in_fields": { + "path_match": "fields\\.[\\d+]$", + "match_pattern": "regex", + "mapping": { + "type": "text", + "index": true, + "norms": false + } + } }, { "string_fields": { "match": "*", "match_mapping_type": "string", "mapping": { - "type": "string", - "index": "analyzed", - "omit_norms": true, + "type": "text", + "index": true, + "norms": false, "fields": { "raw": { - "type": "string", - "index": "not_analyzed", + "type": "keyword", + "index": true, "ignore_above": 256 } } @@ -42,8 +38,8 @@ ], "properties": { "message": { - "type": "string", - "index": "analyzed" + "type": "text", + "index": true }, "exceptions": { "type": "nested", @@ -58,12 +54,12 @@ "type": "integer" }, "StackTraceString": { - "type": "string", - "index": "analyzed" + "type": "text", + "index": true }, "RemoteStackTraceString": { - "type": "string", - "index": "analyzed" + "type": "text", + "index": true }, "ExceptionMessage": { "type": "object", @@ -76,6 +72,9 @@ } } } + }, + "aliases": { + "logstash": {} } } -} \ No newline at end of file +} diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v2.json b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v8_no-aliases_0replicas.json similarity index 59% rename from test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v2.json rename to test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v8_no-aliases_0replicas.json index 6a45a045..e076ad76 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v2.json +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v8_no-aliases_0replicas.json @@ -1,38 +1,35 @@ { - "template": "logstash-*", - "settings": { - "index.refresh_interval": "5s" - }, - "mappings": { - "_default_": { - "_all": { - "enabled": true, - "omit_norms" : true - }, + "index_patterns": [ "logstash-*" ], + "template": { + "settings": { + "index.refresh_interval": "5s", + "number_of_replicas": "0" + }, + "mappings": { "dynamic_templates": [ { - "numerics_in_fields": { - "path_match":"fields\\.[\\d+]$", - "match_pattern":"regex", - "mapping": { - "type":"string", - "index":"analyzed", - "omit_norms":true - } - } + "numerics_in_fields": { + "path_match": "fields\\.[\\d+]$", + "match_pattern": "regex", + "mapping": { + "type": "text", + "index": true, + "norms": false + } + } }, { "string_fields": { "match": "*", "match_mapping_type": "string", "mapping": { - "type": "string", - "index": "analyzed", - "omit_norms": true, + "type": "text", + "index": true, + "norms": false, "fields": { "raw": { - "type": "string", - "index": "not_analyzed", + "type": "keyword", + "index": true, "ignore_above": 256 } } @@ -42,8 +39,8 @@ ], "properties": { "message": { - "type": "string", - "index": "analyzed" + "type": "text", + "index": true }, "exceptions": { "type": "nested", @@ -58,12 +55,12 @@ "type": "integer" }, "StackTraceString": { - "type": "string", - "index": "analyzed" + "type": "text", + "index": true }, "RemoteStackTraceString": { - "type": "string", - "index": "analyzed" + "type": "text", + "index": true }, "ExceptionMessage": { "type": "object", @@ -76,6 +73,8 @@ } } } + }, + "aliases": { } } -} \ No newline at end of file +} diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_5replicas.json b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v8_no-aliases_2shards.json similarity index 58% rename from test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_5replicas.json rename to test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v8_no-aliases_2shards.json index 50c4035f..8caee924 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_5replicas.json +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v8_no-aliases_2shards.json @@ -1,39 +1,36 @@ { - "template": "logstash-*", - "settings": { - "index.refresh_interval": "5s", - "number_of_replicas": "5" - }, - "mappings": { - "_default_": { - "_all": { - "enabled": true, - "omit_norms" : true - }, + "index_patterns": [ "logstash-*" ], + "template": { + "settings": { + "index.refresh_interval": "5s", + "number_of_shards": "2", + "number_of_replicas": "0" + }, + "mappings": { "dynamic_templates": [ { - "numerics_in_fields": { - "path_match":"fields\\.[\\d+]$", - "match_pattern":"regex", - "mapping": { - "type":"string", - "index":"analyzed", - "omit_norms":true - } - } + "numerics_in_fields": { + "path_match": "fields\\.[\\d+]$", + "match_pattern": "regex", + "mapping": { + "type": "text", + "index": true, + "norms": false + } + } }, { "string_fields": { "match": "*", "match_mapping_type": "string", "mapping": { - "type": "string", - "index": "analyzed", - "omit_norms": true, + "type": "text", + "index": true, + "norms": false, "fields": { "raw": { - "type": "string", - "index": "not_analyzed", + "type": "keyword", + "index": true, "ignore_above": 256 } } @@ -43,8 +40,8 @@ ], "properties": { "message": { - "type": "string", - "index": "analyzed" + "type": "text", + "index": true }, "exceptions": { "type": "nested", @@ -59,12 +56,12 @@ "type": "integer" }, "StackTraceString": { - "type": "string", - "index": "analyzed" + "type": "text", + "index": true }, "RemoteStackTraceString": { - "type": "string", - "index": "analyzed" + "type": "text", + "index": true }, "ExceptionMessage": { "type": "object", @@ -77,6 +74,8 @@ } } } + }, + "aliases": { } } -} \ No newline at end of file +} diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_0replicas.json b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v8_no-aliases_5replicas.json similarity index 58% rename from test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_0replicas.json rename to test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v8_no-aliases_5replicas.json index 82b465b4..4b89a3fc 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_0replicas.json +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template_v8_no-aliases_5replicas.json @@ -1,39 +1,35 @@ { - "template": "logstash-*", - "settings": { - "index.refresh_interval": "5s", - "number_of_replicas": "0" - }, - "mappings": { - "_default_": { - "_all": { - "enabled": true, - "omit_norms" : true - }, + "index_patterns": [ "logstash-*" ], + "template": { + "settings": { + "index.refresh_interval": "5s", + "number_of_replicas": "5" + }, + "mappings": { "dynamic_templates": [ { - "numerics_in_fields": { - "path_match":"fields\\.[\\d+]$", - "match_pattern":"regex", - "mapping": { - "type":"string", - "index":"analyzed", - "omit_norms":true - } - } + "numerics_in_fields": { + "path_match": "fields\\.[\\d+]$", + "match_pattern": "regex", + "mapping": { + "type": "text", + "index": true, + "norms": false + } + } }, { "string_fields": { "match": "*", "match_mapping_type": "string", "mapping": { - "type": "string", - "index": "analyzed", - "omit_norms": true, + "type": "text", + "index": true, + "norms": false, "fields": { "raw": { - "type": "string", - "index": "not_analyzed", + "type": "keyword", + "index": true, "ignore_above": 256 } } @@ -43,8 +39,8 @@ ], "properties": { "message": { - "type": "string", - "index": "analyzed" + "type": "text", + "index": true }, "exceptions": { "type": "nested", @@ -59,12 +55,12 @@ "type": "integer" }, "StackTraceString": { - "type": "string", - "index": "analyzed" + "type": "text", + "index": true }, "RemoteStackTraceString": { - "type": "string", - "index": "analyzed" + "type": "text", + "index": true }, "ExceptionMessage": { "type": "object", @@ -77,6 +73,8 @@ } } } + }, + "aliases": { } } -} \ No newline at end of file +} diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/TestDataHelper.cs b/test/Serilog.Sinks.Elasticsearch.Tests/TestDataHelper.cs index dd1b37a1..3f3644a8 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/TestDataHelper.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/TestDataHelper.cs @@ -1,10 +1,7 @@ using System; -using System.Collections.Generic; using System.IO; using System.Linq; using System.Reflection; -using System.Text; -using System.Threading.Tasks; namespace Serilog.Sinks.Elasticsearch.Tests {