From dd75ad8a69284ede49f6f881642dcca53e5099e3 Mon Sep 17 00:00:00 2001 From: Rolf Kristensen Date: Wed, 8 May 2024 16:50:44 +0200 Subject: [PATCH] NLog ElasticSearch Target (#369) * NLog ElasticSearch Target * Allow Test Explorer to run xUnit Tests * NLog ElasticSearch Target (docs) * Server Response can be null in callback * Update README.md * Update README.md about using filebeat as alternative * NLog ElasticsearchTarget assign CloudId switches to NodePoolType.Cloud * NLog ElasticsearchTarget updated unit-test to string-input to force Label-output * NLog ElasticsearchTarget fix integration-test to expect MetaData since enum-value * NLog ElasticsearchTarget fix integration-test to expect MetaData since enum-value * Added GenerateDocumentationFile = true * Renamed NodePoolType to ElasticPoolType * NLog EcsLayout now converts properties of type Enum as string --- ecs-dotnet.sln | 84 ++++-- src/Elastic.CommonSchema.NLog/EcsLayout.cs | 109 ++++--- .../Elastic.CommonSchema.NLog.csproj | 3 +- src/Elastic.CommonSchema.NLog/README.md | 8 +- .../ElasticsearchLoggerProvider.cs | 2 +- .../Elastic.NLog.Targets.csproj | 18 ++ src/Elastic.NLog.Targets/ElasticPoolType.cs | 25 ++ .../ElasticsearchTarget.cs | 285 ++++++++++++++++++ src/Elastic.NLog.Targets/README.md | 115 +++++++ tests-integration/Directory.Build.props | 2 +- ...astic.NLog.Targets.IntegrationTests.csproj | 14 + .../LoggingCluster.cs | 13 + .../LoggingToDataStreamTests.cs | 69 +++++ .../TestBase.cs | 47 +++ tests/Directory.Build.props | 2 +- 15 files changed, 712 insertions(+), 84 deletions(-) create mode 100644 src/Elastic.NLog.Targets/Elastic.NLog.Targets.csproj create mode 100644 src/Elastic.NLog.Targets/ElasticPoolType.cs create mode 100644 src/Elastic.NLog.Targets/ElasticsearchTarget.cs create mode 100644 src/Elastic.NLog.Targets/README.md create mode 100644 tests-integration/Elastic.NLog.Targets.IntegrationTests/Elastic.NLog.Targets.IntegrationTests.csproj create mode 100644 tests-integration/Elastic.NLog.Targets.IntegrationTests/LoggingCluster.cs create mode 100644 tests-integration/Elastic.NLog.Targets.IntegrationTests/LoggingToDataStreamTests.cs create mode 100644 tests-integration/Elastic.NLog.Targets.IntegrationTests/TestBase.cs diff --git a/ecs-dotnet.sln b/ecs-dotnet.sln index d703ab68..52899458 100644 --- a/ecs-dotnet.sln +++ b/ecs-dotnet.sln @@ -33,17 +33,17 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{7610B796-BB3 EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{3582B07D-C2B0-49CC-B676-EAF806EB010E}" ProjectSection(SolutionItems) = preProject - tests\Directory.Build.props = tests\Directory.Build.props tests\.runsettings = tests\.runsettings + tests\Directory.Build.props = tests\Directory.Build.props EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "build", "build", "{441B3323-3651-4B6E-95B3-7CD43E5E223A}" ProjectSection(SolutionItems) = preProject + .ci.runsettings = .ci.runsettings build.bat = build.bat build.sh = build.sh Directory.Build.props = Directory.Build.props global.json = global.json - .ci.runsettings = .ci.runsettings EndProjectSection EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elastic.CommonSchema.Tests", "tests\Elastic.CommonSchema.Tests\Elastic.CommonSchema.Tests.csproj", "{EE4EA2DE-411D-400C-9BF6-8F6AFC17697C}" @@ -76,54 +76,58 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elastic.CommonSchema.Genera EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elastic.Ingest.Elasticsearch.CommonSchema", "src\Elastic.Ingest.Elasticsearch.CommonSchema\Elastic.Ingest.Elasticsearch.CommonSchema.csproj", "{68128AE4-350C-4FB2-A971-C9272A1F3829}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Serilog.Sinks", "src\Elastic.Serilog.Sinks\Elastic.Serilog.Sinks.csproj", "{30080079-D3EE-4BDC-9BE9-9D1B3B2BEF8D}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elastic.Serilog.Sinks", "src\Elastic.Serilog.Sinks\Elastic.Serilog.Sinks.csproj", "{30080079-D3EE-4BDC-9BE9-9D1B3B2BEF8D}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elastic.CommonSchema.Log4net", "src\Elastic.CommonSchema.Log4net\Elastic.CommonSchema.Log4net.csproj", "{DD7D6E56-58DB-4E13-9DFC-AE031F1C31B3}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elastic.CommonSchema.Log4net.Tests", "tests\Elastic.CommonSchema.Log4net.Tests\Elastic.CommonSchema.Log4net.Tests.csproj", "{14BFAF67-8DB6-48D0-B57E-84767BA2A239}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Serilog.Sinks.IntegrationTests", "tests-integration\Elastic.Serilog.Sinks.IntegrationTests\Elastic.Serilog.Sinks.IntegrationTests.csproj", "{622CC10E-B475-4649-8411-CABC31E7C252}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elastic.Serilog.Sinks.IntegrationTests", "tests-integration\Elastic.Serilog.Sinks.IntegrationTests\Elastic.Serilog.Sinks.IntegrationTests.csproj", "{622CC10E-B475-4649-8411-CABC31E7C252}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Apm.Disabled.Serilog.Tests", "tests\Elastic.Apm.Disabled.Serilog.Tests\Elastic.Apm.Disabled.Serilog.Tests.csproj", "{73829D36-DB98-4D8F-8741-F167A787BF7B}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elastic.Apm.Disabled.Serilog.Tests", "tests\Elastic.Apm.Disabled.Serilog.Tests\Elastic.Apm.Disabled.Serilog.Tests.csproj", "{73829D36-DB98-4D8F-8741-F167A787BF7B}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests", "tests-integration\Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests\Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests.csproj", "{1AF36656-7950-42D1-996D-DF5985298926}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests", "tests-integration\Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests\Elastic.Ingest.Elasticsearch.CommonSchema.IntegrationTests.csproj", "{1AF36656-7950-42D1-996D-DF5985298926}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "_solution", "_solution", "{BAF28E09-EAAE-400D-8E0D-6E7C3000997A}" ProjectSection(SolutionItems) = preProject - global.json = global.json .editorconfig = .editorconfig - ecs-dotnet.sln = ecs-dotnet.sln - dotnet-tools.json = dotnet-tools.json - ecs-dotnet.sln.DotSettings = ecs-dotnet.sln.DotSettings - Directory.Build.props = Directory.Build.props - build.sh = build.sh - .pre-commit-config.yaml = .pre-commit-config.yaml .gitattributes = .gitattributes - license.txt = license.txt - build.bat = build.bat - issue_template.md = issue_template.md .gitignore = .gitignore - README.md = README.md + .pre-commit-config.yaml = .pre-commit-config.yaml + build.bat = build.bat + build.sh = build.sh contributing.md = contributing.md - nuget.config = nuget.config + Directory.Build.props = Directory.Build.props + dotnet-tools.json = dotnet-tools.json + ecs-dotnet.sln = ecs-dotnet.sln + ecs-dotnet.sln.DotSettings = ecs-dotnet.sln.DotSettings + global.json = global.json + issue_template.md = issue_template.md + license.txt = license.txt nuget-icon.png = nuget-icon.png + nuget.config = nuget.config + README.md = README.md EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests-integration", "tests-integration", "{947B298F-9139-4868-B337-729541932E4D}" ProjectSection(SolutionItems) = preProject - tests-integration\Directory.Build.props = tests-integration\Directory.Build.props tests-integration\.runsettings = tests-integration\.runsettings + tests-integration\Directory.Build.props = tests-integration\Directory.Build.props EndProjectSection EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elasticsearch.IntegrationDefaults", "tests-integration\Elasticsearch.IntegrationDefaults\Elasticsearch.IntegrationDefaults.csproj", "{AB197BBD-D90D-4ACB-AD09-C59913FA109F}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elasticsearch.IntegrationDefaults", "tests-integration\Elasticsearch.IntegrationDefaults\Elasticsearch.IntegrationDefaults.csproj", "{AB197BBD-D90D-4ACB-AD09-C59913FA109F}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{B268060B-83ED-4944-B135-C362DFCBFC0C}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Serilog.Sinks.Example", "examples\Elastic.Serilog.Sinks.Example\Elastic.Serilog.Sinks.Example.csproj", "{1CAEFBD7-B800-41C4-81D3-CB6839FA563D}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elastic.Serilog.Sinks.Example", "examples\Elastic.Serilog.Sinks.Example\Elastic.Serilog.Sinks.Example.csproj", "{1CAEFBD7-B800-41C4-81D3-CB6839FA563D}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "docs", "docs\docs.csproj", "{7FDB3B31-020A-40E3-B564-F06476320C40}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "docs", "docs\docs.csproj", "{7FDB3B31-020A-40E3-B564-F06476320C40}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "aspnetcore-with-extensions-logging", "examples\aspnetcore-with-extensions-logging\aspnetcore-with-extensions-logging.csproj", "{D866F335-BC19-49A8-AF72-4BA66CC7AFFB}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "aspnetcore-with-extensions-logging", "examples\aspnetcore-with-extensions-logging\aspnetcore-with-extensions-logging.csproj", "{D866F335-BC19-49A8-AF72-4BA66CC7AFFB}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elastic.NLog.Targets", "src\Elastic.NLog.Targets\Elastic.NLog.Targets.csproj", "{692F8035-F3F9-4714-8C9D-D54AF4CEB0E0}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elastic.NLog.Targets.IntegrationTests", "tests-integration\Elastic.NLog.Targets.IntegrationTests\Elastic.NLog.Targets.IntegrationTests.csproj", "{D1C3CAFB-A59D-4E3F-ADD1-4CB281E5349D}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "playground", "examples\playground\playground.csproj", "{86AEB76A-C210-4250-8541-B349C26C1683}" EndProject @@ -133,6 +137,10 @@ Global Release|Any CPU = Release|Any CPU EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution + {80D7CE12-D0C9-44E2-9BF9-5762D52ADA05}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {80D7CE12-D0C9-44E2-9BF9-5762D52ADA05}.Debug|Any CPU.Build.0 = Debug|Any CPU + {80D7CE12-D0C9-44E2-9BF9-5762D52ADA05}.Release|Any CPU.ActiveCfg = Release|Any CPU + {80D7CE12-D0C9-44E2-9BF9-5762D52ADA05}.Release|Any CPU.Build.0 = Release|Any CPU {70072EAB-C5DF-4100-B594-B9DC3169D604}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {70072EAB-C5DF-4100-B594-B9DC3169D604}.Debug|Any CPU.Build.0 = Debug|Any CPU {70072EAB-C5DF-4100-B594-B9DC3169D604}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -227,10 +235,6 @@ Global {30080079-D3EE-4BDC-9BE9-9D1B3B2BEF8D}.Debug|Any CPU.Build.0 = Debug|Any CPU {30080079-D3EE-4BDC-9BE9-9D1B3B2BEF8D}.Release|Any CPU.ActiveCfg = Release|Any CPU {30080079-D3EE-4BDC-9BE9-9D1B3B2BEF8D}.Release|Any CPU.Build.0 = Release|Any CPU - {622CC10E-B475-4649-8411-CABC31E7C252}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {622CC10E-B475-4649-8411-CABC31E7C252}.Debug|Any CPU.Build.0 = Debug|Any CPU - {622CC10E-B475-4649-8411-CABC31E7C252}.Release|Any CPU.ActiveCfg = Release|Any CPU - {622CC10E-B475-4649-8411-CABC31E7C252}.Release|Any CPU.Build.0 = Release|Any CPU {DD7D6E56-58DB-4E13-9DFC-AE031F1C31B3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {DD7D6E56-58DB-4E13-9DFC-AE031F1C31B3}.Debug|Any CPU.Build.0 = Debug|Any CPU {DD7D6E56-58DB-4E13-9DFC-AE031F1C31B3}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -239,6 +243,10 @@ Global {14BFAF67-8DB6-48D0-B57E-84767BA2A239}.Debug|Any CPU.Build.0 = Debug|Any CPU {14BFAF67-8DB6-48D0-B57E-84767BA2A239}.Release|Any CPU.ActiveCfg = Release|Any CPU {14BFAF67-8DB6-48D0-B57E-84767BA2A239}.Release|Any CPU.Build.0 = Release|Any CPU + {622CC10E-B475-4649-8411-CABC31E7C252}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {622CC10E-B475-4649-8411-CABC31E7C252}.Debug|Any CPU.Build.0 = Debug|Any CPU + {622CC10E-B475-4649-8411-CABC31E7C252}.Release|Any CPU.ActiveCfg = Release|Any CPU + {622CC10E-B475-4649-8411-CABC31E7C252}.Release|Any CPU.Build.0 = Release|Any CPU {73829D36-DB98-4D8F-8741-F167A787BF7B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {73829D36-DB98-4D8F-8741-F167A787BF7B}.Debug|Any CPU.Build.0 = Debug|Any CPU {73829D36-DB98-4D8F-8741-F167A787BF7B}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -261,6 +269,14 @@ Global {D866F335-BC19-49A8-AF72-4BA66CC7AFFB}.Debug|Any CPU.Build.0 = Debug|Any CPU {D866F335-BC19-49A8-AF72-4BA66CC7AFFB}.Release|Any CPU.ActiveCfg = Release|Any CPU {D866F335-BC19-49A8-AF72-4BA66CC7AFFB}.Release|Any CPU.Build.0 = Release|Any CPU + {692F8035-F3F9-4714-8C9D-D54AF4CEB0E0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {692F8035-F3F9-4714-8C9D-D54AF4CEB0E0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {692F8035-F3F9-4714-8C9D-D54AF4CEB0E0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {692F8035-F3F9-4714-8C9D-D54AF4CEB0E0}.Release|Any CPU.Build.0 = Release|Any CPU + {D1C3CAFB-A59D-4E3F-ADD1-4CB281E5349D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D1C3CAFB-A59D-4E3F-ADD1-4CB281E5349D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D1C3CAFB-A59D-4E3F-ADD1-4CB281E5349D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D1C3CAFB-A59D-4E3F-ADD1-4CB281E5349D}.Release|Any CPU.Build.0 = Release|Any CPU {86AEB76A-C210-4250-8541-B349C26C1683}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {86AEB76A-C210-4250-8541-B349C26C1683}.Debug|Any CPU.Build.0 = Debug|Any CPU {86AEB76A-C210-4250-8541-B349C26C1683}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -270,6 +286,7 @@ Global HideSolutionNode = FALSE EndGlobalSection GlobalSection(NestedProjects) = preSolution + {80D7CE12-D0C9-44E2-9BF9-5762D52ADA05} = {441B3323-3651-4B6E-95B3-7CD43E5E223A} {70072EAB-C5DF-4100-B594-B9DC3169D604} = {7610B796-BB3E-4CB2-8296-79BBFF6D23FC} {45BC8315-6AD6-4F3C-B590-7B52D19ED401} = {7610B796-BB3E-4CB2-8296-79BBFF6D23FC} {D7BA6070-909F-402E-A6F4-1CE54A7BE0B7} = {3582B07D-C2B0-49CC-B676-EAF806EB010E} @@ -282,28 +299,29 @@ Global {4E0D951B-FEC5-4043-913A-BED795892405} = {3582B07D-C2B0-49CC-B676-EAF806EB010E} {EE4EA2DE-411D-400C-9BF6-8F6AFC17697C} = {3582B07D-C2B0-49CC-B676-EAF806EB010E} {6BE3084A-D84D-4782-9915-6E41575712C7} = {7610B796-BB3E-4CB2-8296-79BBFF6D23FC} + {4138E98A-4714-4139-BD89-D9FF4F2A3A73} = {947B298F-9139-4868-B337-729541932E4D} {0881CC2E-BFBB-40DB-BA5B-B3D23A985F73} = {7610B796-BB3E-4CB2-8296-79BBFF6D23FC} {89ADA999-1A1D-4B51-8CEE-39A553F669D1} = {3582B07D-C2B0-49CC-B676-EAF806EB010E} {03FD4BFA-F9A5-4C16-ACA1-30FD060DFAEA} = {9F103D76-F7FA-4D10-8214-6E79C28D5AEC} {9F103D76-F7FA-4D10-8214-6E79C28D5AEC} = {05075402-8669-45BD-913A-BD40A29BBEAB} + {EC19A9E1-79CC-46A8-94D7-EE66ED22D3BD} = {B268060B-83ED-4944-B135-C362DFCBFC0C} {D88AAA7D-1AEE-4B4C-BE37-69BA85DA07DA} = {7610B796-BB3E-4CB2-8296-79BBFF6D23FC} + {0E7008E1-B215-4B9B-BC28-DC9D31415FB9} = {947B298F-9139-4868-B337-729541932E4D} {F319AD28-A0A4-4012-8480-E2A4CFA755C2} = {05075402-8669-45BD-913A-BD40A29BBEAB} {D87AE73E-8112-444C-8F2F-CFBC4F738026} = {05075402-8669-45BD-913A-BD40A29BBEAB} - {80D7CE12-D0C9-44E2-9BF9-5762D52ADA05} = {441B3323-3651-4B6E-95B3-7CD43E5E223A} + {D6F0D170-39D7-4868-86EE-990B6B05C14D} = {B268060B-83ED-4944-B135-C362DFCBFC0C} {68128AE4-350C-4FB2-A971-C9272A1F3829} = {7610B796-BB3E-4CB2-8296-79BBFF6D23FC} {30080079-D3EE-4BDC-9BE9-9D1B3B2BEF8D} = {7610B796-BB3E-4CB2-8296-79BBFF6D23FC} {DD7D6E56-58DB-4E13-9DFC-AE031F1C31B3} = {7610B796-BB3E-4CB2-8296-79BBFF6D23FC} {14BFAF67-8DB6-48D0-B57E-84767BA2A239} = {3582B07D-C2B0-49CC-B676-EAF806EB010E} + {622CC10E-B475-4649-8411-CABC31E7C252} = {947B298F-9139-4868-B337-729541932E4D} {73829D36-DB98-4D8F-8741-F167A787BF7B} = {3582B07D-C2B0-49CC-B676-EAF806EB010E} - {0E7008E1-B215-4B9B-BC28-DC9D31415FB9} = {947B298F-9139-4868-B337-729541932E4D} {1AF36656-7950-42D1-996D-DF5985298926} = {947B298F-9139-4868-B337-729541932E4D} - {622CC10E-B475-4649-8411-CABC31E7C252} = {947B298F-9139-4868-B337-729541932E4D} - {4138E98A-4714-4139-BD89-D9FF4F2A3A73} = {947B298F-9139-4868-B337-729541932E4D} {AB197BBD-D90D-4ACB-AD09-C59913FA109F} = {947B298F-9139-4868-B337-729541932E4D} - {D6F0D170-39D7-4868-86EE-990B6B05C14D} = {B268060B-83ED-4944-B135-C362DFCBFC0C} - {EC19A9E1-79CC-46A8-94D7-EE66ED22D3BD} = {B268060B-83ED-4944-B135-C362DFCBFC0C} {1CAEFBD7-B800-41C4-81D3-CB6839FA563D} = {05075402-8669-45BD-913A-BD40A29BBEAB} {D866F335-BC19-49A8-AF72-4BA66CC7AFFB} = {05075402-8669-45BD-913A-BD40A29BBEAB} + {692F8035-F3F9-4714-8C9D-D54AF4CEB0E0} = {7610B796-BB3E-4CB2-8296-79BBFF6D23FC} + {D1C3CAFB-A59D-4E3F-ADD1-4CB281E5349D} = {947B298F-9139-4868-B337-729541932E4D} {86AEB76A-C210-4250-8541-B349C26C1683} = {05075402-8669-45BD-913A-BD40A29BBEAB} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution diff --git a/src/Elastic.CommonSchema.NLog/EcsLayout.cs b/src/Elastic.CommonSchema.NLog/EcsLayout.cs index 30cf65bf..d824040c 100644 --- a/src/Elastic.CommonSchema.NLog/EcsLayout.cs +++ b/src/Elastic.CommonSchema.NLog/EcsLayout.cs @@ -135,35 +135,35 @@ private static bool NLogWeb4Registered() => public Layout DisableThreadAgnostic => IncludeScopeProperties ? _disableThreadAgnostic : null; // ReSharper restore UnusedMember.Global - /// + /// public Layout AgentId { get; set; } - /// + /// public Layout AgentName { get; set; } - /// + /// public Layout AgentType { get; set; } - /// + /// public Layout AgentVersion { get; set; } // ReSharper disable AutoPropertyCanBeMadeGetOnly.Global - /// + /// public Layout ApmTraceId { get; set; } - /// + /// public Layout ApmTransactionId { get; set; } - /// + /// public Layout ApmSpanId { get; set; } - /// + /// public Layout ApmServiceName { get; set; } - /// + /// public Layout ApmServiceNodeName { get; set; } - /// + /// public Layout ApmServiceVersion { get; set; } - /// + /// public Layout LogOriginCallSiteMethod { get; set; } - /// + /// public Layout LogOriginCallSiteFile { get; set; } - /// + /// public Layout LogOriginCallSiteLine { get; set; } /// @@ -181,11 +181,11 @@ private static bool NLogWeb4Registered() => /// public Layout EventDurationMs { get; set; } - /// + /// public Layout HostId { get; set; } - /// + /// public Layout HostIp { get; set; } - /// + /// public Layout HostName { get; set; } /// @@ -220,48 +220,48 @@ private static bool NLogWeb4Registered() => /// public Layout MessageTemplate { get; set; } - /// + /// public Layout ProcessExecutable { get; set; } - /// + /// public Layout ProcessId { get; set; } - /// + /// public Layout ProcessName { get; set; } - /// + /// public Layout ProcessThreadId { get; set; } - /// + /// public Layout ProcessThreadName { get; set; } - /// + /// public Layout ProcessTitle { get; set; } - /// + /// public Layout ServerAddress { get; set; } - /// + /// public Layout ServerIp { get; set; } - /// + /// public Layout ServerUser { get; set; } - /// + /// public Layout HttpRequestId { get; set; } - /// + /// public Layout HttpRequestMethod { get; set; } - /// + /// public Layout RequestBodyBytes { get; set; } - /// + /// public Layout HttpRequestReferrer { get; set; } - /// + /// public Layout HttpResponseStatusCode { get; set; } - /// + /// public Layout UrlScheme { get; set; } - /// + /// public Layout UrlDomain { get; set; } - /// + /// public Layout UrlPort { get; set; } - /// + /// public Layout UrlPath { get; set; } - /// + /// public Layout UrlQuery { get; set; } - /// + /// public Layout UrlUserName { get; set; } /// @@ -283,6 +283,15 @@ private static bool NLogWeb4Registered() => /// protected override void RenderFormattedMessage(LogEventInfo logEvent, StringBuilder target) + { + var ecsDocument = RenderEcsDocument(logEvent); + ecsDocument.Serialize(target); + } + + /// + /// Create an instance of and enrich it with as many fields as possible. + /// + public NLogEcsDocument RenderEcsDocument(LogEventInfo logEvent) { var ecsEvent = EcsDocument.CreateNewWithDefaults(logEvent.TimeStamp, logEvent.Exception, NlogEcsDocumentCreationOptions.Default); @@ -293,7 +302,8 @@ protected override void RenderFormattedMessage(LogEventInfo logEvent, StringBuil // prefer setting service information set by Elastic APM var service = GetService(logEvent); - if (service != null) ecsEvent.Service = service; + if (service != null) + ecsEvent.Service = service; ecsEvent.Message = logEvent.FormattedMessage; ecsEvent.Log = GetLog(logEvent); @@ -322,8 +332,7 @@ protected override void RenderFormattedMessage(LogEventInfo logEvent, StringBuil EnrichEvent(logEvent, ref ecsDocument); //Allow programmatic actions to enrich before serializing EnrichAction?.Invoke(ecsDocument, logEvent); - - ecsDocument.Serialize(target); + return ecsEvent; } private Service GetService(LogEventInfo logEventInfo) @@ -375,9 +384,7 @@ private MetadataDictionary GetMetadata(LogEventInfo e) continue; var propertyValue = prop.Value; - if (propertyValue is null or IConvertible || propertyValue.GetType().IsValueType) - Populate(metadata, propertyName, propertyValue); - else + if (!TryPopulateWhenSafe(metadata, propertyName, propertyValue)) { templateParameters ??= e.MessageTemplateParameters; var value = AllowSerializePropertyValue(propertyName, templateParameters) ? propertyValue : propertyValue.ToString(); @@ -394,7 +401,10 @@ private MetadataDictionary GetMetadata(LogEventInfo e) continue; var propertyValue = MappedDiagnosticsLogicalContext.GetObject(key); - Populate(metadata, key, propertyValue); + if (!TryPopulateWhenSafe(metadata, key, propertyValue)) + { + Populate(metadata, key, propertyValue.ToString()); + } } } @@ -716,6 +726,19 @@ private static long GetSysLogSeverity(LogLevel logLevel) return 2; // LogLevel.Fatal } + private static bool TryPopulateWhenSafe(IDictionary propertyBag, string key, object value) + { + if (value is null or IConvertible || value.GetType().IsValueType) + { + if (value is Enum) + value = value.ToString(); + Populate(propertyBag, key, value); + return true; + } + + return false; + } + private static void Populate(IDictionary propertyBag, string key, object value) { if (string.IsNullOrEmpty(key)) diff --git a/src/Elastic.CommonSchema.NLog/Elastic.CommonSchema.NLog.csproj b/src/Elastic.CommonSchema.NLog/Elastic.CommonSchema.NLog.csproj index cf4b2a56..cdf9a625 100644 --- a/src/Elastic.CommonSchema.NLog/Elastic.CommonSchema.NLog.csproj +++ b/src/Elastic.CommonSchema.NLog/Elastic.CommonSchema.NLog.csproj @@ -1,10 +1,11 @@ - + netstandard2.0;netstandard2.1;net461 Elastic Common Schema (ECS) NLog Layout NLog Layout that formats log events in accordance with Elastic Common Schema (ECS). True + true diff --git a/src/Elastic.CommonSchema.NLog/README.md b/src/Elastic.CommonSchema.NLog/README.md index da20add7..1069eadd 100644 --- a/src/Elastic.CommonSchema.NLog/README.md +++ b/src/Elastic.CommonSchema.NLog/README.md @@ -131,11 +131,10 @@ An example of the output is given below: "message": "Info \"X\" 2.2", "ecs.version": "8.6.0", "log": { - "logger": "Elastic.CommonSchema.NLog.Tests.LogTestsBase", + "logger": "Elastic.CommonSchema.NLog.Tests.LogTestsBase" }, "labels": { - "ValueX": "X", - "MessageTemplate": "Info {ValueX} {SomeY} {NotX}" + "ValueX": "X" }, "agent": { "type": "Elastic.CommonSchema.NLog", @@ -165,7 +164,8 @@ An example of the output is given below: }, "metadata": { "SomeY": 2.2 - } + }, + "MessageTemplate": "Info {ValueX} {SomeY} {NotX}" } ``` diff --git a/src/Elastic.Extensions.Logging/ElasticsearchLoggerProvider.cs b/src/Elastic.Extensions.Logging/ElasticsearchLoggerProvider.cs index 1a25d7a9..4bfe68e7 100644 --- a/src/Elastic.Extensions.Logging/ElasticsearchLoggerProvider.cs +++ b/src/Elastic.Extensions.Logging/ElasticsearchLoggerProvider.cs @@ -99,7 +99,7 @@ private static NodePool CreateNodePool(ElasticsearchLoggerOptions loggerOptions) case NodePoolType.Static: return new StaticNodePool(nodeUris); case NodePoolType.Sticky: - return new StaticNodePool(nodeUris); + return new StickyNodePool(nodeUris); // case NodePoolType.StickySniffing: case NodePoolType.Cloud: if (shipTo.CloudId.IsNullOrEmpty()) diff --git a/src/Elastic.NLog.Targets/Elastic.NLog.Targets.csproj b/src/Elastic.NLog.Targets/Elastic.NLog.Targets.csproj new file mode 100644 index 00000000..bc8c9086 --- /dev/null +++ b/src/Elastic.NLog.Targets/Elastic.NLog.Targets.csproj @@ -0,0 +1,18 @@ + + + + netstandard2.0 + Elasticsearch NLog Target + NLog Target that exports directly to Elastic Cloud or individual Elasticsearch nodes + NLog.Targets + True + enable + true + + + + + + + + diff --git a/src/Elastic.NLog.Targets/ElasticPoolType.cs b/src/Elastic.NLog.Targets/ElasticPoolType.cs new file mode 100644 index 00000000..5dd5d1c6 --- /dev/null +++ b/src/Elastic.NLog.Targets/ElasticPoolType.cs @@ -0,0 +1,25 @@ +using Elastic.Transport; + +namespace NLog.Targets +{ + /// + /// The type of connection pool for Elasticsearch + /// + public enum ElasticPoolType + { + /// Not configured + Unknown = 0, + /// + SingleNode, + /// + Sniffing, + /// + Static, + /// + Sticky, + /// + StickySniffing, + /// + Cloud + } +} diff --git a/src/Elastic.NLog.Targets/ElasticsearchTarget.cs b/src/Elastic.NLog.Targets/ElasticsearchTarget.cs new file mode 100644 index 00000000..c43e6236 --- /dev/null +++ b/src/Elastic.NLog.Targets/ElasticsearchTarget.cs @@ -0,0 +1,285 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Elastic.Channels; +using Elastic.Channels.Buffers; +using Elastic.Channels.Diagnostics; +using Elastic.Ingest.Elasticsearch; +using Elastic.Ingest.Elasticsearch.CommonSchema; +using Elastic.Ingest.Elasticsearch.DataStreams; +using Elastic.Ingest.Elasticsearch.Serialization; +using Elastic.Transport; +using Elastic.Transport.Products.Elasticsearch; +using NLog.Layouts; +using static Elastic.CommonSchema.NLog.EcsLayout; + +namespace NLog.Targets +{ + /// + /// NLog target for writing logs directly to Elasticsearch or Elastic Cloud + /// + [Target("Elasticsearch")] + public class ElasticsearchTarget : TargetWithLayout + { + /// + public override Layout Layout { get => _layout; set => _layout = value as Elastic.CommonSchema.NLog.EcsLayout ?? _layout; } + private Elastic.CommonSchema.NLog.EcsLayout _layout = new Elastic.CommonSchema.NLog.EcsLayout(); + private EcsDataStreamChannel? _channel; + + /// + /// Gets or sets the connection pool type. Default for multiple nodes is Sniffing; other supported values are + /// Static, Sticky, or force to SingleNode. + /// + public ElasticPoolType NodePoolType { get; set; } + + /// + /// Gets or sets the URIs of the Elasticsearch nodes in the connection pool. If not specified the default single node + /// "http://localhost:9200" is used. + /// + public Layout? NodeUris { get; set; } + + /// + /// Allows the target datastream to be bootstrapped. The default is no bootstrapping + /// since we assume the configured user might not have management privileges + /// + public BootstrapMethod BootstrapMethod { get; set; } + + /// Generic type describing the data + public Layout? DataStreamType { get; set; } = "logs"; + /// Describes the data ingested and its structure + public Layout? DataStreamSet { get; set; } = "dotnet"; + /// User-configurable arbitrary grouping + public Layout? DataStreamNamespace { get; set; } = "default"; + + /// + /// The maximum number of in flight instances that can be queued in memory. If this threshold is reached, events will be dropped + /// Defaults to 100_000 + /// + public int InboundBufferMaxSize { get; set; } + + /// + /// The maximum size to export to at once. + /// Defaults to 1_000 + /// + public int OutboundBufferMaxSize { get; set; } + + /// + /// The maximum lifetime of a buffer to export to . + /// If a buffer is older then the configured it will be flushed to + /// regardless of it's current size + /// Defaults to 5 seconds + /// + public int OutboundBufferMaxLifetimeSeconds { get; set; } + + /// + /// The maximum number of consumers allowed to poll for new events on the channel. + /// Defaults to 1, increase to introduce concurrency. + /// + public int ExportMaxConcurrency { get; set; } + + /// + /// The times to retry an export if yields items to retry. + /// Whether or not items are selected for retrying depends on the actual channel implementation + /// Defaults to 3, when yields any items + /// + public int ExportMaxRetries { get; set; } = -1; + + /// + /// The ILM Policy to apply, see the following for more details: + /// https://www.elastic.co/guide/en/elasticsearch/reference/current/index-lifecycle-management.html + /// Defaults to `logs` which is shipped by default with Elasticsearch + /// + public Layout? IlmPolicy { get; set; } + + /// + /// Gets or sets the cloud ID, where connection pool type is Cloud. + /// + public Layout? CloudId + { + get => _cloudId; + set + { + _cloudId = value; + if (NodePoolType == ElasticPoolType.Unknown && value != null) + NodePoolType = ElasticPoolType.Cloud; + } + } + private Layout? _cloudId; + + /// + /// Gets or sets the API Key, where connection pool type is Cloud, and authenticating via API Key. + /// + public Layout? ApiKey { get; set; } + + /// + /// Gets or sets the password, where connection pool type is Cloud, and authenticating via username/password. + /// + public Layout? Password { get; set; } + + /// + /// Gets or sets the username, where connection pool type is Cloud, and authenticating via username/password. + /// + public Layout? Username { get; set; } + + /// + /// Provide callbacks to further configure + /// + public Action>? ConfigureChannel { get; set; } + + /// + public IChannelDiagnosticsListener? DiagnosticsListener => _channel?.DiagnosticsListener; + + /// + protected override void InitializeTarget() + { + var ilmPolicy = IlmPolicy?.Render(LogEventInfo.CreateNullEvent()); + var dataStreamType = DataStreamType?.Render(LogEventInfo.CreateNullEvent()) ?? string.Empty; + var dataStreamSet = DataStreamSet?.Render(LogEventInfo.CreateNullEvent()) ?? string.Empty; + var dataStreamNamespace = DataStreamNamespace?.Render(LogEventInfo.CreateNullEvent()) ?? string.Empty; + + var connectionPool = CreateNodePool(); + var config = new TransportConfiguration(connectionPool, productRegistration: ElasticsearchProductRegistration.Default); + // Cloud sets authentication as required parameter in the constructor + if (NodePoolType != ElasticPoolType.Cloud) + config = SetAuthenticationOnTransport(config); + + var transport = new DistributedTransport(config); + var channelOptions = new DataStreamChannelOptions(transport) + { + DataStream = new DataStreamName(dataStreamType, dataStreamSet, dataStreamNamespace), + WriteEvent = async (stream, ctx, logEvent) => await logEvent.SerializeAsync(stream, ctx).ConfigureAwait(false), + }; + if (InboundBufferMaxSize > 0) + channelOptions.BufferOptions.InboundBufferMaxSize = InboundBufferMaxSize; + if (OutboundBufferMaxSize > 0) + channelOptions.BufferOptions.OutboundBufferMaxSize = OutboundBufferMaxSize; + if (OutboundBufferMaxLifetimeSeconds > 0) + channelOptions.BufferOptions.OutboundBufferMaxLifetime = TimeSpan.FromSeconds(OutboundBufferMaxLifetimeSeconds); + if (ExportMaxConcurrency > 0) + channelOptions.BufferOptions.ExportMaxConcurrency = ExportMaxConcurrency; + if (ExportMaxRetries >= 0) + channelOptions.BufferOptions.ExportMaxRetries = ExportMaxRetries; + ConfigureChannel?.Invoke(channelOptions); + + var channel = new EcsDataStreamChannel(channelOptions, new[] { new InternalLoggerCallbackListener() }); + channel.BootstrapElasticsearch(BootstrapMethod, ilmPolicy); + _channel = channel; + } + + /// + protected override void CloseTarget() + { + _channel?.Dispose(); + base.CloseTarget(); + } + + /// + protected override void Write(LogEventInfo logEvent) + { + var ecsDoc = _layout.RenderEcsDocument(logEvent); + _channel?.TryWrite(ecsDoc); + } + + private NodePool CreateNodePool() + { + var nodeUris = NodeUris?.Render(LogEventInfo.CreateNullEvent()).Split(new[] { ',' }).Select(uri => uri.Trim()).Where(uri => !string.IsNullOrEmpty(uri)).Select(uri => new Uri(uri)).ToArray() ?? Array.Empty(); + if (nodeUris.Length == 0 && NodePoolType != ElasticPoolType.Cloud) + return new SingleNodePool(new Uri("http://localhost:9200")); + if (NodePoolType == ElasticPoolType.SingleNode || NodePoolType == ElasticPoolType.Unknown && nodeUris.Length == 1) + return new SingleNodePool(nodeUris[0]); + + switch (NodePoolType) + { + case ElasticPoolType.Unknown: + case ElasticPoolType.Sniffing: + return new SniffingNodePool(nodeUris); + case ElasticPoolType.Static: + return new StaticNodePool(nodeUris); + case ElasticPoolType.Sticky: + return new StickyNodePool(nodeUris); + case ElasticPoolType.Cloud: + var cloudId = CloudId?.Render(LogEventInfo.CreateNullEvent()) ?? string.Empty; + if (string.IsNullOrEmpty(cloudId)) + throw new NLogConfigurationException($"ElasticSearch Cloud {nameof(CloudNodePool)} requires '{nameof(CloudId)}' to be provided as well"); + + var apiKey = ApiKey?.Render(LogEventInfo.CreateNullEvent()) ?? string.Empty; + if (!string.IsNullOrEmpty(apiKey)) + { + var apiKeyCredentials = new ApiKey(apiKey); + return new CloudNodePool(cloudId, apiKeyCredentials); + } + + var username = Username?.Render(LogEventInfo.CreateNullEvent()) ?? string.Empty; + var password = Password?.Render(LogEventInfo.CreateNullEvent()) ?? string.Empty; + if (!string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(password)) + { + var basicAuthCredentials = new BasicAuthentication(username, password); + return new CloudNodePool(cloudId, basicAuthCredentials); + } + + throw new NLogConfigurationException($"ElasticSearch Cloud requires either '{nameof(ApiKey)}' or" + + $"'{nameof(Username)}' and '{nameof(Password)}"); + //case ElasticPoolType.StickySniffing: + default: + throw new NLogConfigurationException($"Unrecognised ElasticSearch connection pool type '{NodePoolType}' specified in the configuration.", + nameof(NodePoolType)); + } + } + + private TransportConfiguration SetAuthenticationOnTransport(TransportConfiguration config) + { + var apiKey = ApiKey?.Render(LogEventInfo.CreateNullEvent()) ?? string.Empty; + var username = Username?.Render(LogEventInfo.CreateNullEvent()) ?? string.Empty; + var password = Password?.Render(LogEventInfo.CreateNullEvent()) ?? string.Empty; + if (!string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(password)) + config = config.Authentication(new BasicAuthentication(username, password)); + else if (!string.IsNullOrEmpty(apiKey)) + config = config.Authentication(new ApiKey(apiKey)); + return config; + } + } + + internal class InternalLoggerCallbackListener : IChannelCallbacks where TNLogEcsDocument : NLogEcsDocument, new() + { + public Action? ExportExceptionCallback { get; } + public Action? ExportResponseCallback { get; } + + // ReSharper disable UnassignedGetOnlyAutoProperty + public Action? ExportItemsAttemptCallback { get; } + public Action>? ExportMaxRetriesCallback { get; } + public Action>? ExportRetryCallback { get; } + public Action? PublishToInboundChannelCallback { get; } + public Action? PublishToInboundChannelFailureCallback { get; } + public Action? PublishToOutboundChannelCallback { get; } + public Action? OutboundChannelStartedCallback { get; } + public Action? OutboundChannelExitedCallback { get; } + public Action? InboundChannelStartedCallback { get; } + public Action? PublishToOutboundChannelFailureCallback { get; } + public Action? ExportBufferCallback { get; } + public Action? ExportRetryableCountCallback { get; } + // ReSharper enable UnassignedGetOnlyAutoProperty + + public InternalLoggerCallbackListener() + { + ExportExceptionCallback = ex => + { + NLog.Common.InternalLogger.Error(ex, "ElasticSearch - Export Exception"); + }; + ExportResponseCallback = (response, _) => + { + if (response is null) + return; + + if (response.TryGetElasticsearchServerError(out var error)) + NLog.Common.InternalLogger.Error("ElasticSearch - Export Response Server Error - {0}", error); + + if (response.Items?.Count > 0) + { + foreach (var itemResult in response.Items) + if (itemResult?.Status >= 300) + NLog.Common.InternalLogger.Error("ElasticSearch - Export Item failed to {0} document status {1} - {2}", itemResult.Action, itemResult.Status, itemResult.Error); + } + }; + } + } +} diff --git a/src/Elastic.NLog.Targets/README.md b/src/Elastic.NLog.Targets/README.md new file mode 100644 index 00000000..dd15a9e1 --- /dev/null +++ b/src/Elastic.NLog.Targets/README.md @@ -0,0 +1,115 @@ +# Elastic.NLog.Targets + +A [NLog](https://nlog-project.org/) target that writes logs directly to [Elasticsearch](https://www.elastic.co/elasticsearch/) or [Elastic Cloud](https://www.elastic.co/cloud/) + +## Packages + +The .NET assemblies are published to NuGet under the package name [Elastic.NLog.Targets](http://nuget.org/packages/Elastic.NLog.Targets) + +## How to use from API + +```csharp +var config = new LoggingConfiguration(); +var elasticTarget = new ElasticsearchTarget("elastic") { Layout = new EcsLayout(), NodesUri = "http://localhost:9200" }; +config.AddRule(LogLevel.Debug, LogLevel.Fatal, elasticTarget); +LogManager.Configuration = config; +var logger = LogManager.GetCurrentClassLogger(); +``` + +## How to use from NLog.config + +```xml + + + + + + + + + + + + + + + + + +``` + + +## ElasticsearchTarget Parameter Options + +* **Export Destination** + - _NodePoolType_ - Connection pool type + - SingleNode - Pool with single node or endpoint + - Sniffing - Pool with Supports-Reseeding + - Static - Pool without Supports-Reseeding + - Sticky - Pool without Supports-Reseeding and stays on the first node. + - StickySniffing - Pool with Supports-Reseeding and stays on the first node. + - Cloud - Pool seeded with CloudId + - _NodeUris_ - URIs of the Elasticsearch nodes in the connection pool (comma delimited) + - _CloudId_ - When using NodePoolType = Cloud + +* **Export Authentication** + - _ApiKey_ - When using NodePoolType = Cloud and authentication via API key. + - _Username_ - When basic authenticating via username/password. + - _Password_ - When basic authenticating via username/password. + +* **Export Buffering** + - _InboundBufferMaxSize_ - Max number of in flight instances that can be queued in memory. Default = 100000 + - _OutboundBufferMaxSize_ - Max size to export. Default = 1000 + - _OutboundBufferMaxLifetimeSeconds_ - Maximum lifetime of a buffer to export in seconds. Default = 5 sec + - _ExportMaxConcurrency_ - Max number of consumers allowed to poll for new events on the channel. Default = 1 + - _ExportMaxRetries_ - Max number of times to retry an export. Default = 3 + +* **Export DataStream** + - _DataStreamType_ - Generic type describing the data. Defaults = 'logs' + - _DataStreamSet_ - Describes the data ingested and its structure. Default = 'dotnet' + - _DataStreamNamespace_ - User-configurable arbitrary grouping. Default = 'default' + +Notice that export depends on in-memory queue, that is lost on application-crash / -exit. +If higher gurantee of delivery is required, then consider using [Elastic.CommonSchema.NLog](https://www.nuget.org/packages/Elastic.CommonSchema.NLog) +together with NLog FileTarget and use [filebeat](https://www.elastic.co/beats/filebeat) to ship these logs. + +Check out [Elastic Agent & Fleet](https://www.elastic.co/guide/en/fleet/current/fleet-overview.html) to simplify collecting logs and metrics on the edge. + +## ElasticsearchTarget Layout Configuration + +NLog Layout allows one to configure NLog Target options from environment. + +**Lookup NodeUris from appsettings.json** +```xml + +``` + +Example appsettings.json on .NET Core: +```json + { + "ConnectionStrings": { + "ElasticSearch": "http://localhost:9200" + } + } +``` + +**Lookup NodeUris from app.config** +```xml + +``` + +Example app.config on .NET Framework: +```xml + + + + + +``` + +**Lookup ConnectionString from environment-variable** +```xml + +``` \ No newline at end of file diff --git a/tests-integration/Directory.Build.props b/tests-integration/Directory.Build.props index 1492e789..7c016357 100644 --- a/tests-integration/Directory.Build.props +++ b/tests-integration/Directory.Build.props @@ -14,7 +14,7 @@ - + diff --git a/tests-integration/Elastic.NLog.Targets.IntegrationTests/Elastic.NLog.Targets.IntegrationTests.csproj b/tests-integration/Elastic.NLog.Targets.IntegrationTests/Elastic.NLog.Targets.IntegrationTests.csproj new file mode 100644 index 00000000..7790d7d4 --- /dev/null +++ b/tests-integration/Elastic.NLog.Targets.IntegrationTests/Elastic.NLog.Targets.IntegrationTests.csproj @@ -0,0 +1,14 @@ + + + + net6.0 + false + NLog.Targets.Elastic.IntegrationTests + + + + + + + + diff --git a/tests-integration/Elastic.NLog.Targets.IntegrationTests/LoggingCluster.cs b/tests-integration/Elastic.NLog.Targets.IntegrationTests/LoggingCluster.cs new file mode 100644 index 00000000..0e934853 --- /dev/null +++ b/tests-integration/Elastic.NLog.Targets.IntegrationTests/LoggingCluster.cs @@ -0,0 +1,13 @@ +using Elasticsearch.IntegrationDefaults; +using Xunit; + +[assembly: TestFramework("Elastic.Elasticsearch.Xunit.Sdk.ElasticTestFramework", "Elastic.Elasticsearch.Xunit")] + +namespace NLog.Targets.Elastic.IntegrationTests; + +/// Declare our cluster that we want to inject into our test classes +public class LoggingCluster : TestClusterBase +{ + public LoggingCluster() : base(9201) { } + +} diff --git a/tests-integration/Elastic.NLog.Targets.IntegrationTests/LoggingToDataStreamTests.cs b/tests-integration/Elastic.NLog.Targets.IntegrationTests/LoggingToDataStreamTests.cs new file mode 100644 index 00000000..91ca0cf4 --- /dev/null +++ b/tests-integration/Elastic.NLog.Targets.IntegrationTests/LoggingToDataStreamTests.cs @@ -0,0 +1,69 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Elastic.Channels.Diagnostics; +using Elastic.Clients.Elasticsearch; +using Elastic.CommonSchema; +using FluentAssertions; +using Xunit; +using Xunit.Abstractions; + +namespace NLog.Targets.Elastic.IntegrationTests +{ + public class LoggingToDataStreamTests : TestBase + { + public LoggingToDataStreamTests(LoggingCluster cluster, ITestOutputHelper output) : base(cluster, output) { } + + private IDisposable CreateLogger( + out NLog.Logger logger, + out NLog.LogFactory logFactory, + out string @namespace, + out WaitHandle waitHandle, + out IChannelDiagnosticsListener listener + ) => + base.CreateLogger(out logger, out logFactory, out @namespace, out waitHandle, out listener, (cfg) => + { + cfg.DataStreamType = "x"; + cfg.DataStreamSet = "dotnet"; + var nodesUris = string.Join(",", Client.ElasticsearchClientSettings.NodePool.Nodes.Select(n => n.Uri.ToString()).ToArray()); + cfg.NodeUris = nodesUris; + cfg.NodePoolType = ElasticPoolType.Static; + }); + + // ReSharper disable once UnusedMember.Local + private enum MyEnum { Success, Failure } + + [Fact] + public async Task LogsEndUpInCluster() + { + using var _ = CreateLogger(out var logger, out var provider, out var @namespace, out var waitHandle, out var listener); + var dataStream = $"x-dotnet-{@namespace}"; + + logger.Error("an error occurred {Status}", MyEnum.Failure); + + if (!waitHandle.WaitOne(TimeSpan.FromSeconds(10))) + throw new Exception($"No flush occurred in 10 seconds: {listener}", listener.ObservedException); + + listener.PublishSuccess.Should().BeTrue("{0}", listener); + listener.ObservedException.Should().BeNull(); + + await Client.Indices.RefreshAsync(dataStream); + + var response = Client.Search(new SearchRequest(dataStream)); + + response.IsValidResponse.Should().BeTrue("{0}", response.DebugInformation); + response.Total.Should().BeGreaterThan(0); + + var loggedError = response.Documents.First(); + loggedError.Message.Should().Be("an error occurred Failure"); + loggedError.Log.Should().NotBeNull(); + loggedError.Log.Level.Should().Be("Error"); + loggedError.Ecs.Version.Should().Be(EcsDocument.Version); + loggedError.Ecs.Version.Should().NotStartWith("v"); + + loggedError.Labels.Should().ContainKey("Status"); + loggedError.Labels["Status"].Should().Be("Failure"); + } + } +} diff --git a/tests-integration/Elastic.NLog.Targets.IntegrationTests/TestBase.cs b/tests-integration/Elastic.NLog.Targets.IntegrationTests/TestBase.cs new file mode 100644 index 00000000..13ac356e --- /dev/null +++ b/tests-integration/Elastic.NLog.Targets.IntegrationTests/TestBase.cs @@ -0,0 +1,47 @@ +using System; +using System.Linq; +using System.Threading; +using Elastic.Channels.Diagnostics; +using Elastic.Clients.Elasticsearch; +using Elastic.Elasticsearch.Xunit.XunitPlumbing; +using Xunit.Abstractions; + +namespace NLog.Targets.Elastic.IntegrationTests; + +public abstract class TestBase : IClusterFixture +{ + protected ElasticsearchClient Client { get; } + + protected TestBase(LoggingCluster cluster, ITestOutputHelper output) => + Client = cluster.CreateClient(output); + + protected IDisposable CreateLogger( + out NLog.Logger logger, + out NLog.LogFactory logFactory, + out string @namespace, + out WaitHandle waitHandle, + out IChannelDiagnosticsListener listener, + Action setupTarget + ) + { + var slim = new CountdownEvent(1); + waitHandle = slim.WaitHandle; + @namespace = Guid.NewGuid().ToString("N").ToLowerInvariant().Substring(0, 6); + + logFactory = new NLog.LogFactory(); + var logConfig = new NLog.Config.LoggingConfiguration(logFactory); + var logTarget = new NLog.Targets.ElasticsearchTarget() { Name = "elastic" }; + logTarget.DataStreamNamespace = @namespace; + logTarget.OutboundBufferMaxSize = 1; + logTarget.OutboundBufferMaxLifetimeSeconds = 1; + logTarget.ExportMaxRetries = 0; + logTarget.ExportMaxConcurrency = 1; + logTarget.ConfigureChannel = (cfg) => cfg.BufferOptions.WaitHandle = slim; + setupTarget?.Invoke(logTarget); + logConfig.AddRuleForAllLevels(logTarget); + logFactory.Configuration = logConfig; + listener = logTarget.DiagnosticsListener; + logger = logFactory.GetLogger("TestLogger"); + return logFactory; + } +} diff --git a/tests/Directory.Build.props b/tests/Directory.Build.props index 87156b89..df36a9e8 100644 --- a/tests/Directory.Build.props +++ b/tests/Directory.Build.props @@ -14,7 +14,7 @@ - +