Merge pull request #1600 from kube-logging/syslog-ng-date-parser

syslog-ng: add source date-parser

charts/logging-operator/crds/logging.banzaicloud.io_loggings.yaml

@@ -15288,6 +15288,13 @@ spec:
                     type: object
                   skipRBACCreate:
                     type: boolean
+                  sourceDateParser:
+                    properties:
+                      format:
+                        type: string
+                      template:
+                        type: string
+                    type: object
                   sourceMetrics:
                     items:
                       properties:

config/crd/bases/logging.banzaicloud.io_loggings.yaml

@@ -15288,6 +15288,13 @@ spec:
                     type: object
                   skipRBACCreate:
                     type: boolean
+                  sourceDateParser:
+                    properties:
+                      format:
+                        type: string
+                      template:
+                        type: string
+                    type: object
                   sourceMetrics:
                     items:
                       properties:

config/samples/syslog-ng-simple.yaml

@@ -0,0 +1,46 @@
+kind: Logging
+apiVersion: logging.banzaicloud.io/v1beta1
+metadata:
+  name: logging
+spec:
+  controlNamespace: default
+  fluentbit: {}
+  syslogNG:
+    globalOptions:
+      log_level: trace
+---
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: SyslogNGFlow
+metadata:
+  name: all1
+spec:
+  match: {}
+  localOutputRefs:
+    - http
+    - http2
+---
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: SyslogNGFlow
+metadata:
+  name: all2
+spec:
+  match: {}
+  localOutputRefs:
+    - http
+    - http2
+---
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: SyslogNGOutput
+metadata:
+  name: http
+spec:
+  file:
+    path: "/tmp/log"
+---
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: SyslogNGOutput
+metadata:
+  name: http2
+spec:
+  file:
+    path: "/tmp/log"

docs/configuration/crds/v1beta1/syslogng_types.md

@@ -50,6 +50,11 @@ SyslogNGSpec defines the desired state of SyslogNG
 ### skipRBACCreate (bool, optional) {#syslogngspec-skiprbaccreate}
 
 
+### sourceDateParser (*SourceDateParser, optional) {#syslogngspec-sourcedateparser}
+
+Parses date automatically from the timestamp registered by the container runtime. Note: json key prefix and delimiter are respected 
+
+
 ### sourceMetrics ([]filter.MetricsProbe, optional) {#syslogngspec-sourcemetrics}
 
 
@@ -60,6 +65,19 @@ SyslogNGSpec defines the desired state of SyslogNG
 
 
 
+## SourceDateParser
+
+### format (*string, optional) {#sourcedateparser-format}
+
+Default: "%FT%T.%f%z" 
+
+
+### template (*string, optional) {#sourcedateparser-template}
+
+Default(depending on JSONKeyPrefix): "${json.time}" 
+
+
+
 ## SyslogNGTLS
 
 SyslogNGTLS defines the TLS configs

docs/syslog-ng-date-parser.md

@@ -0,0 +1,29 @@
+## syslog-ng date parser
+
+By default, the syslog-ng aggregator uses the time when a message has been received on its input source as the timestamp.
+In case we want to use the timestamp written in the message metadata, we should use a [date-parser](https://axoflow.com/docs/axosyslog-core/chapter-parsers/date-parser/date-parser-options/).
+
+To enable the timestamps written by the container runtime (_cri_ or _docker_) and parsed by fluentbit automatically, we just
+have to define the `sourceDateParser` in the _syslog-ng_ spec.
+
+```
+kind: Logging
+metadata:
+  name: example
+spec:
+  syslogNG:
+    sourceDateParser: {}
+```
+
+In case we want to define our own parser format and template we can also do so (these are the default values):
+
+```
+kind: Logging
+metadata:
+  name: example
+spec:
+  syslogNG:
+    sourceDateParser:
+      format: "%FT%T.%f%z"
+      template: "${json.time}"
+```

pkg/sdk/logging/api/v1beta1/syslogng_types.go

@@ -47,13 +47,22 @@ type SyslogNGSpec struct {
 	GlobalOptions                       *GlobalOptions               `json:"globalOptions,omitempty"`
 	JSONKeyPrefix                       string                       `json:"jsonKeyPrefix,omitempty"`
 	JSONKeyDelimiter                    string                       `json:"jsonKeyDelim,omitempty"`
-	MaxConnections                      int                          `json:"maxConnections,omitempty"`
-	LogIWSize                           int                          `json:"logIWSize,omitempty"`
-	SourceMetrics                       []filter.MetricsProbe        `json:"sourceMetrics,omitempty"`
-
+	// Parses date automatically from the timestamp registered by the container runtime.
+	// Note: json key prefix and delimiter are respected
+	SourceDateParser *SourceDateParser     `json:"sourceDateParser,omitempty"`
+	MaxConnections   int                   `json:"maxConnections,omitempty"`
+	LogIWSize        int                   `json:"logIWSize,omitempty"`
+	SourceMetrics    []filter.MetricsProbe `json:"sourceMetrics,omitempty"`
 	// TODO: option to turn on/off buffer volume PVC
 }
 
+type SourceDateParser struct {
+	// Default: "%FT%T.%f%z"
+	Format *string `json:"format,omitempty"`
+	// Default(depending on JSONKeyPrefix): "${json.time}"
+	Template *string `json:"template,omitempty"`
+}
+
 // +kubebuilder:object:generate=true
 
 // SyslogNGTLS defines the TLS configs

pkg/sdk/logging/model/syslogng/config/config.go

@@ -15,6 +15,7 @@
 package config
 
 import (
+	"fmt"
 	"io"
 	"reflect"
 
@@ -122,6 +123,20 @@ func configRenderer(in Input) (render.Renderer, error) {
 				}),
 			}, nil),
 	}
+
+	if in.Logging.Spec.SyslogNGSpec.SourceDateParser != nil {
+		setDefault(&in.Logging.Spec.SyslogNGSpec.SourceDateParser.Format, amp("%FT%T.%f%z"))
+		setDefault(&in.Logging.Spec.SyslogNGSpec.SourceDateParser.Template,
+			amp(fmt.Sprintf("${%stime}", in.Logging.Spec.SyslogNGSpec.JSONKeyPrefix)))
+		sourceParsers = append(sourceParsers, renderDriver(
+			Field{
+				Value: reflect.ValueOf(DateParser{
+					Format:   *in.Logging.Spec.SyslogNGSpec.SourceDateParser.Format,
+					Template: *in.Logging.Spec.SyslogNGSpec.SourceDateParser.Template,
+				}),
+			}, nil))
+	}
+
 	for _, sm := range in.Logging.Spec.SyslogNGSpec.SourceMetrics {
 		if sm.Labels == nil {
 			sm.Labels = make(filter.ArrowMap, 0)

pkg/sdk/logging/model/syslogng/config/config_test.go

@@ -19,6 +19,7 @@ import (
 	"testing"
 
 	"github.com/cisco-open/operator-tools/pkg/secret"
+	"github.com/cisco-open/operator-tools/pkg/utils"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -1012,6 +1013,75 @@ source "main_input" {
         };
     };
 };
+`),
+		},
+		"date-parser default": {
+			input: Input{
+				Logging: v1beta1.Logging{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace: "logging",
+						Name:      "test",
+					},
+					Spec: v1beta1.LoggingSpec{
+						SyslogNGSpec: &v1beta1.SyslogNGSpec{
+							SourceDateParser: &v1beta1.SourceDateParser{},
+						},
+					},
+				},
+				SecretLoaderFactory: &TestSecretLoaderFactory{},
+				SourcePort:          601,
+			},
+			wantOut: Untab(`@version: current
+
+@include "scl.conf"
+
+source "main_input" {
+    channel {
+        source {
+            network(flags("no-parse") port(601) transport("tcp"));
+        };
+        parser {
+            json-parser(prefix("json."));
+            date-parser(format("%FT%T.%f%z") template("${json.time}"));
+        };
+    };
+};
+`),
+		},
+		"date-parser custom": {
+			input: Input{
+				Logging: v1beta1.Logging{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace: "logging",
+						Name:      "test",
+					},
+					Spec: v1beta1.LoggingSpec{
+						SyslogNGSpec: &v1beta1.SyslogNGSpec{
+							SourceDateParser: &v1beta1.SourceDateParser{
+								Format:   utils.StringPointer("asd"),
+								Template: utils.StringPointer("bsd"),
+							},
+						},
+					},
+				},
+				SecretLoaderFactory: &TestSecretLoaderFactory{},
+				SourcePort:          601,
+			},
+			wantOut: Untab(`@version: current
+
+@include "scl.conf"
+
+source "main_input" {
+    channel {
+        source {
+            network(flags("no-parse") port(601) transport("tcp"));
+        };
+        parser {
+            json-parser(prefix("json."));
+            date-parser(format("asd") template("bsd"));
+        };
+    };
+};
 `),
 		},
 	}

pkg/sdk/logging/model/syslogng/config/output_tests/mongodb_test.go

@@ -19,12 +19,13 @@ import (
 	"testing"
 
 	"github.com/cisco-open/operator-tools/pkg/secret"
-	"github.com/kube-logging/logging-operator/pkg/sdk/logging/api/v1beta1"
-	"github.com/kube-logging/logging-operator/pkg/sdk/logging/model/syslogng/config"
-	"github.com/kube-logging/logging-operator/pkg/sdk/logging/model/syslogng/output"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/kube-logging/logging-operator/pkg/sdk/logging/api/v1beta1"
+	"github.com/kube-logging/logging-operator/pkg/sdk/logging/model/syslogng/config"
+	"github.com/kube-logging/logging-operator/pkg/sdk/logging/model/syslogng/output"
 )
 
 func TestMongoDBOutput(t *testing.T) {

pkg/sdk/logging/model/syslogng/config/output_tests/redis_test.go

@@ -19,12 +19,13 @@ import (
 	"testing"
 
 	"github.com/cisco-open/operator-tools/pkg/secret"
-	"github.com/kube-logging/logging-operator/pkg/sdk/logging/api/v1beta1"
-	"github.com/kube-logging/logging-operator/pkg/sdk/logging/model/syslogng/config"
-	"github.com/kube-logging/logging-operator/pkg/sdk/logging/model/syslogng/output"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/kube-logging/logging-operator/pkg/sdk/logging/api/v1beta1"
+	"github.com/kube-logging/logging-operator/pkg/sdk/logging/model/syslogng/config"
+	"github.com/kube-logging/logging-operator/pkg/sdk/logging/model/syslogng/output"
 )
 
 func TestRedisOutput(t *testing.T) {

pkg/sdk/logging/model/syslogng/config/output_tests/s3_test.go

@@ -19,12 +19,13 @@ import (
 	"testing"
 
 	"github.com/cisco-open/operator-tools/pkg/secret"
-	"github.com/kube-logging/logging-operator/pkg/sdk/logging/api/v1beta1"
-	"github.com/kube-logging/logging-operator/pkg/sdk/logging/model/syslogng/config"
-	"github.com/kube-logging/logging-operator/pkg/sdk/logging/model/syslogng/output"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/kube-logging/logging-operator/pkg/sdk/logging/api/v1beta1"
+	"github.com/kube-logging/logging-operator/pkg/sdk/logging/model/syslogng/config"
+	"github.com/kube-logging/logging-operator/pkg/sdk/logging/model/syslogng/output"
 )
 
 func TestS3OutputMinimal(t *testing.T) {

pkg/sdk/logging/model/syslogng/config/parser.go

@@ -38,3 +38,9 @@ func isActiveParserDriver(f Field) bool {
 func hasParserDriverTag(f Field) bool {
 	return structFieldSettings(f.Meta).Has("parser-drv")
 }
+
+type DateParser struct {
+	__meta   struct{} `syslog-ng:"name=date-parser"` //lint:ignore U1000 field used for adding tag to the type
+	Format   string   `syslog-ng:"name=format,optional"`
+	Template string   `syslog-ng:"name=template,optional"`
+}