From 2bcb0867497c48ca8a7febd2bd8285a940289ab8 Mon Sep 17 00:00:00 2001 From: Brett Jones Date: Wed, 12 Aug 2020 14:16:00 -0500 Subject: [PATCH 1/2] Bug 1852341: fix legacy syslog k8s Metadata The `use_record true` setting is trying to use the "level" from the k8s audit log which [is Metadata](https://kubernetes.io/docs/tasks/debug-application-cluster/audit/#audit-policy). I believe we need to relabel k8s audit logs "level" field to "k8s_audit_level" so that fluentd doesn't try to use it as the severity and we can retain the original information. --- pkg/generators/forwarding/fluentd/templates.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/generators/forwarding/fluentd/templates.go b/pkg/generators/forwarding/fluentd/templates.go index f7a7afd7b1..08b00eeec7 100644 --- a/pkg/generators/forwarding/fluentd/templates.go +++ b/pkg/generators/forwarding/fluentd/templates.go @@ -210,6 +210,14 @@ const fluentConfTemplate = `{{- define "fluentConf" -}} remove_keys req,res,msg,name,level,v,pid,err + + @type record_transformer + + k8s_audit_level ${record['level']} + + remove_keys level + + @type viaq_data_model elasticsearch_index_prefix_field 'viaq_index_name' From 1ec23387d3db727af734438825cdfaea5dbca45b Mon Sep 17 00:00:00 2001 From: Brett Jones Date: Thu, 13 Aug 2020 10:49:00 -0500 Subject: [PATCH 2/2] fix unit tests --- .../forwarding/fluentd/fluent_conf_test.go | 26 +++++++++++++++++++ .../fluentd/output_conf_legacy_test.go | 24 +++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/pkg/generators/forwarding/fluentd/fluent_conf_test.go b/pkg/generators/forwarding/fluentd/fluent_conf_test.go index 8c3a3d75e0..504c683406 100644 --- a/pkg/generators/forwarding/fluentd/fluent_conf_test.go +++ b/pkg/generators/forwarding/fluentd/fluent_conf_test.go @@ -352,6 +352,15 @@ var _ = Describe("Generating fluentd config", func() { remove_keys req,res,msg,name,level,v,pid,err + + + @type record_transformer + + k8s_audit_level ${record['level']} + + remove_keys level + + @type viaq_data_model elasticsearch_index_prefix_field 'viaq_index_name' @@ -765,6 +774,15 @@ var _ = Describe("Generating fluentd config", func() { remove_keys req,res,msg,name,level,v,pid,err + + + @type record_transformer + + k8s_audit_level ${record['level']} + + remove_keys level + + @type viaq_data_model elasticsearch_index_prefix_field 'viaq_index_name' @@ -1564,6 +1582,14 @@ var _ = Describe("Generating fluentd config", func() { remove_keys req,res,msg,name,level,v,pid,err + + @type record_transformer + + k8s_audit_level ${record['level']} + + remove_keys level + + @type viaq_data_model elasticsearch_index_prefix_field 'viaq_index_name' diff --git a/pkg/generators/forwarding/fluentd/output_conf_legacy_test.go b/pkg/generators/forwarding/fluentd/output_conf_legacy_test.go index 91273049da..8196ba4e30 100644 --- a/pkg/generators/forwarding/fluentd/output_conf_legacy_test.go +++ b/pkg/generators/forwarding/fluentd/output_conf_legacy_test.go @@ -299,6 +299,14 @@ var _ = Describe("Generating fluentd legacy output store config blocks", func() remove_keys req,res,msg,name,level,v,pid,err + + @type record_transformer + + k8s_audit_level ${record['level']} + + remove_keys level + + @type viaq_data_model elasticsearch_index_prefix_field 'viaq_index_name' @@ -739,6 +747,14 @@ var _ = Describe("Generating fluentd legacy output store config blocks", func() remove_keys req,res,msg,name,level,v,pid,err + + @type record_transformer + + k8s_audit_level ${record['level']} + + remove_keys level + + @type viaq_data_model elasticsearch_index_prefix_field 'viaq_index_name' @@ -1180,6 +1196,14 @@ var _ = Describe("Generating fluentd legacy output store config blocks", func() remove_keys req,res,msg,name,level,v,pid,err + + @type record_transformer + + k8s_audit_level ${record['level']} + + remove_keys level + + @type viaq_data_model elasticsearch_index_prefix_field 'viaq_index_name'