diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_loggings.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_loggings.yaml index 29628b155..ebefd4ea6 100644 --- a/charts/logging-operator/crds/logging.banzaicloud.io_loggings.yaml +++ b/charts/logging-operator/crds/logging.banzaicloud.io_loggings.yaml @@ -15288,6 +15288,13 @@ spec: type: object skipRBACCreate: type: boolean + sourceDateParser: + properties: + format: + type: string + template: + type: string + type: object sourceMetrics: items: properties: diff --git a/config/crd/bases/logging.banzaicloud.io_loggings.yaml b/config/crd/bases/logging.banzaicloud.io_loggings.yaml index 29628b155..ebefd4ea6 100644 --- a/config/crd/bases/logging.banzaicloud.io_loggings.yaml +++ b/config/crd/bases/logging.banzaicloud.io_loggings.yaml @@ -15288,6 +15288,13 @@ spec: type: object skipRBACCreate: type: boolean + sourceDateParser: + properties: + format: + type: string + template: + type: string + type: object sourceMetrics: items: properties: diff --git a/config/samples/syslog-ng.yaml b/config/samples/syslog-ng-pvc-metrics.yaml similarity index 100% rename from config/samples/syslog-ng.yaml rename to config/samples/syslog-ng-pvc-metrics.yaml diff --git a/config/samples/syslog-ng-simple.yaml b/config/samples/syslog-ng-simple.yaml new file mode 100644 index 000000000..6c99b14a3 --- /dev/null +++ b/config/samples/syslog-ng-simple.yaml @@ -0,0 +1,46 @@ +kind: Logging +apiVersion: logging.banzaicloud.io/v1beta1 +metadata: + name: logging +spec: + controlNamespace: default + fluentbit: {} + syslogNG: + globalOptions: + log_level: trace +--- +apiVersion: logging.banzaicloud.io/v1beta1 +kind: SyslogNGFlow +metadata: + name: all1 +spec: + match: {} + localOutputRefs: + - http + - http2 +--- +apiVersion: logging.banzaicloud.io/v1beta1 +kind: SyslogNGFlow +metadata: + name: all2 +spec: + match: {} + localOutputRefs: + - http + - http2 +--- +apiVersion: logging.banzaicloud.io/v1beta1 +kind: SyslogNGOutput +metadata: + name: http +spec: + file: + path: "/tmp/log" +--- +apiVersion: logging.banzaicloud.io/v1beta1 +kind: SyslogNGOutput +metadata: + name: http2 +spec: + file: + path: "/tmp/log" diff --git a/docs/configuration/crds/v1beta1/syslogng_types.md b/docs/configuration/crds/v1beta1/syslogng_types.md index 3ee329c6c..b0ef8ad90 100644 --- a/docs/configuration/crds/v1beta1/syslogng_types.md +++ b/docs/configuration/crds/v1beta1/syslogng_types.md @@ -50,6 +50,11 @@ SyslogNGSpec defines the desired state of SyslogNG ### skipRBACCreate (bool, optional) {#syslogngspec-skiprbaccreate} +### sourceDateParser (*SourceDateParser, optional) {#syslogngspec-sourcedateparser} + +Parses date automatically from the timestamp registered by the container runtime. Note: json key prefix and delimiter are respected + + ### sourceMetrics ([]filter.MetricsProbe, optional) {#syslogngspec-sourcemetrics} @@ -60,6 +65,19 @@ SyslogNGSpec defines the desired state of SyslogNG +## SourceDateParser + +### format (*string, optional) {#sourcedateparser-format} + +Default: "%FT%T.%f%z" + + +### template (*string, optional) {#sourcedateparser-template} + +Default(depending on JSONKeyPrefix): "${json.time}" + + + ## SyslogNGTLS SyslogNGTLS defines the TLS configs diff --git a/docs/syslog-ng-date-parser.md b/docs/syslog-ng-date-parser.md new file mode 100644 index 000000000..ffce447e0 --- /dev/null +++ b/docs/syslog-ng-date-parser.md @@ -0,0 +1,29 @@ +## syslog-ng date parser + +By default, the syslog-ng aggregator uses the time when a message has been received on its input source as the timestamp. +In case we want to use the timestamp written in the message metadata, we should use a [date-parser](https://axoflow.com/docs/axosyslog-core/chapter-parsers/date-parser/date-parser-options/). + +To enable the timestamps written by the container runtime (_cri_ or _docker_) and parsed by fluentbit automatically, we just +have to define the `sourceDateParser` in the _syslog-ng_ spec. + +``` +kind: Logging +metadata: + name: example +spec: + syslogNG: + sourceDateParser: {} +``` + +In case we want to define our own parser format and template we can also do so (these are the default values): + +``` +kind: Logging +metadata: + name: example +spec: + syslogNG: + sourceDateParser: + format: "%FT%T.%f%z" + template: "${json.time}" +``` diff --git a/pkg/sdk/logging/api/v1beta1/syslogng_types.go b/pkg/sdk/logging/api/v1beta1/syslogng_types.go index 068331fdc..b3390afac 100644 --- a/pkg/sdk/logging/api/v1beta1/syslogng_types.go +++ b/pkg/sdk/logging/api/v1beta1/syslogng_types.go @@ -47,13 +47,22 @@ type SyslogNGSpec struct { GlobalOptions *GlobalOptions `json:"globalOptions,omitempty"` JSONKeyPrefix string `json:"jsonKeyPrefix,omitempty"` JSONKeyDelimiter string `json:"jsonKeyDelim,omitempty"` - MaxConnections int `json:"maxConnections,omitempty"` - LogIWSize int `json:"logIWSize,omitempty"` - SourceMetrics []filter.MetricsProbe `json:"sourceMetrics,omitempty"` - + // Parses date automatically from the timestamp registered by the container runtime. + // Note: json key prefix and delimiter are respected + SourceDateParser *SourceDateParser `json:"sourceDateParser,omitempty"` + MaxConnections int `json:"maxConnections,omitempty"` + LogIWSize int `json:"logIWSize,omitempty"` + SourceMetrics []filter.MetricsProbe `json:"sourceMetrics,omitempty"` // TODO: option to turn on/off buffer volume PVC } +type SourceDateParser struct { + // Default: "%FT%T.%f%z" + Format *string `json:"format,omitempty"` + // Default(depending on JSONKeyPrefix): "${json.time}" + Template *string `json:"template,omitempty"` +} + // +kubebuilder:object:generate=true // SyslogNGTLS defines the TLS configs diff --git a/pkg/sdk/logging/api/v1beta1/zz_generated.deepcopy.go b/pkg/sdk/logging/api/v1beta1/zz_generated.deepcopy.go index d44b9784c..c45b639ee 100644 --- a/pkg/sdk/logging/api/v1beta1/zz_generated.deepcopy.go +++ b/pkg/sdk/logging/api/v1beta1/zz_generated.deepcopy.go @@ -2576,6 +2576,31 @@ func (in *ServiceMonitorConfig) DeepCopy() *ServiceMonitorConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SourceDateParser) DeepCopyInto(out *SourceDateParser) { + *out = *in + if in.Format != nil { + in, out := &in.Format, &out.Format + *out = new(string) + **out = **in + } + if in.Template != nil { + in, out := &in.Template, &out.Template + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SourceDateParser. +func (in *SourceDateParser) DeepCopy() *SourceDateParser { + if in == nil { + return nil + } + out := new(SourceDateParser) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Stats) DeepCopyInto(out *Stats) { *out = *in @@ -3241,6 +3266,11 @@ func (in *SyslogNGSpec) DeepCopyInto(out *SyslogNGSpec) { *out = new(GlobalOptions) (*in).DeepCopyInto(*out) } + if in.SourceDateParser != nil { + in, out := &in.SourceDateParser, &out.SourceDateParser + *out = new(SourceDateParser) + (*in).DeepCopyInto(*out) + } if in.SourceMetrics != nil { in, out := &in.SourceMetrics, &out.SourceMetrics *out = make([]syslogngfilter.MetricsProbe, len(*in)) diff --git a/pkg/sdk/logging/model/syslogng/config/config.go b/pkg/sdk/logging/model/syslogng/config/config.go index a1788c378..85bc77a7d 100644 --- a/pkg/sdk/logging/model/syslogng/config/config.go +++ b/pkg/sdk/logging/model/syslogng/config/config.go @@ -15,6 +15,7 @@ package config import ( + "fmt" "io" "reflect" @@ -122,6 +123,20 @@ func configRenderer(in Input) (render.Renderer, error) { }), }, nil), } + + if in.Logging.Spec.SyslogNGSpec.SourceDateParser != nil { + setDefault(&in.Logging.Spec.SyslogNGSpec.SourceDateParser.Format, amp("%FT%T.%f%z")) + setDefault(&in.Logging.Spec.SyslogNGSpec.SourceDateParser.Template, + amp(fmt.Sprintf("${%stime}", in.Logging.Spec.SyslogNGSpec.JSONKeyPrefix))) + sourceParsers = append(sourceParsers, renderDriver( + Field{ + Value: reflect.ValueOf(DateParser{ + Format: *in.Logging.Spec.SyslogNGSpec.SourceDateParser.Format, + Template: *in.Logging.Spec.SyslogNGSpec.SourceDateParser.Template, + }), + }, nil)) + } + for _, sm := range in.Logging.Spec.SyslogNGSpec.SourceMetrics { if sm.Labels == nil { sm.Labels = make(filter.ArrowMap, 0) diff --git a/pkg/sdk/logging/model/syslogng/config/config_test.go b/pkg/sdk/logging/model/syslogng/config/config_test.go index 1426ec2c6..5b0008b61 100644 --- a/pkg/sdk/logging/model/syslogng/config/config_test.go +++ b/pkg/sdk/logging/model/syslogng/config/config_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/cisco-open/operator-tools/pkg/secret" + "github.com/cisco-open/operator-tools/pkg/utils" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -1012,6 +1013,75 @@ source "main_input" { }; }; }; +`), + }, + "date-parser default": { + input: Input{ + Logging: v1beta1.Logging{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "logging", + Name: "test", + }, + Spec: v1beta1.LoggingSpec{ + SyslogNGSpec: &v1beta1.SyslogNGSpec{ + SourceDateParser: &v1beta1.SourceDateParser{}, + }, + }, + }, + SecretLoaderFactory: &TestSecretLoaderFactory{}, + SourcePort: 601, + }, + wantOut: Untab(`@version: current + +@include "scl.conf" + +source "main_input" { + channel { + source { + network(flags("no-parse") port(601) transport("tcp")); + }; + parser { + json-parser(prefix("json.")); + date-parser(format("%FT%T.%f%z") template("${json.time}")); + }; + }; +}; +`), + }, + "date-parser custom": { + input: Input{ + Logging: v1beta1.Logging{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "logging", + Name: "test", + }, + Spec: v1beta1.LoggingSpec{ + SyslogNGSpec: &v1beta1.SyslogNGSpec{ + SourceDateParser: &v1beta1.SourceDateParser{ + Format: utils.StringPointer("asd"), + Template: utils.StringPointer("bsd"), + }, + }, + }, + }, + SecretLoaderFactory: &TestSecretLoaderFactory{}, + SourcePort: 601, + }, + wantOut: Untab(`@version: current + +@include "scl.conf" + +source "main_input" { + channel { + source { + network(flags("no-parse") port(601) transport("tcp")); + }; + parser { + json-parser(prefix("json.")); + date-parser(format("asd") template("bsd")); + }; + }; +}; `), }, } diff --git a/pkg/sdk/logging/model/syslogng/config/output_tests/mongodb_test.go b/pkg/sdk/logging/model/syslogng/config/output_tests/mongodb_test.go index 4e112d89e..f7bbae7fb 100644 --- a/pkg/sdk/logging/model/syslogng/config/output_tests/mongodb_test.go +++ b/pkg/sdk/logging/model/syslogng/config/output_tests/mongodb_test.go @@ -19,12 +19,13 @@ import ( "testing" "github.com/cisco-open/operator-tools/pkg/secret" - "github.com/kube-logging/logging-operator/pkg/sdk/logging/api/v1beta1" - "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/syslogng/config" - "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/syslogng/output" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/kube-logging/logging-operator/pkg/sdk/logging/api/v1beta1" + "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/syslogng/config" + "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/syslogng/output" ) func TestMongoDBOutput(t *testing.T) { diff --git a/pkg/sdk/logging/model/syslogng/config/output_tests/redis_test.go b/pkg/sdk/logging/model/syslogng/config/output_tests/redis_test.go index 7c2531ac2..7e0973c60 100644 --- a/pkg/sdk/logging/model/syslogng/config/output_tests/redis_test.go +++ b/pkg/sdk/logging/model/syslogng/config/output_tests/redis_test.go @@ -19,12 +19,13 @@ import ( "testing" "github.com/cisco-open/operator-tools/pkg/secret" - "github.com/kube-logging/logging-operator/pkg/sdk/logging/api/v1beta1" - "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/syslogng/config" - "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/syslogng/output" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/kube-logging/logging-operator/pkg/sdk/logging/api/v1beta1" + "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/syslogng/config" + "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/syslogng/output" ) func TestRedisOutput(t *testing.T) { diff --git a/pkg/sdk/logging/model/syslogng/config/output_tests/s3_test.go b/pkg/sdk/logging/model/syslogng/config/output_tests/s3_test.go index e4d57c28b..50c1a2091 100644 --- a/pkg/sdk/logging/model/syslogng/config/output_tests/s3_test.go +++ b/pkg/sdk/logging/model/syslogng/config/output_tests/s3_test.go @@ -19,12 +19,13 @@ import ( "testing" "github.com/cisco-open/operator-tools/pkg/secret" - "github.com/kube-logging/logging-operator/pkg/sdk/logging/api/v1beta1" - "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/syslogng/config" - "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/syslogng/output" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/kube-logging/logging-operator/pkg/sdk/logging/api/v1beta1" + "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/syslogng/config" + "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/syslogng/output" ) func TestS3OutputMinimal(t *testing.T) { diff --git a/pkg/sdk/logging/model/syslogng/config/parser.go b/pkg/sdk/logging/model/syslogng/config/parser.go index 2ad9a57df..b9e2a0897 100644 --- a/pkg/sdk/logging/model/syslogng/config/parser.go +++ b/pkg/sdk/logging/model/syslogng/config/parser.go @@ -38,3 +38,9 @@ func isActiveParserDriver(f Field) bool { func hasParserDriverTag(f Field) bool { return structFieldSettings(f.Meta).Has("parser-drv") } + +type DateParser struct { + __meta struct{} `syslog-ng:"name=date-parser"` //lint:ignore U1000 field used for adding tag to the type + Format string `syslog-ng:"name=format,optional"` + Template string `syslog-ng:"name=template,optional"` +}