From 6b94111519316ce70a616d697c8dc7eb19cdf0e1 Mon Sep 17 00:00:00 2001 From: ChrsMark Date: Mon, 3 Jun 2024 19:02:33 +0300 Subject: [PATCH 1/2] [pkg/stanza] container: move k8s metadata into resource attributes Signed-off-by: ChrsMark --- .../container_parser_resource_attributes.yaml | 28 +++++++ pkg/stanza/docs/operators/container.md | 78 ++++++++++++------- .../operator/parser/container/parser.go | 2 +- .../operator/parser/container/parser_test.go | 54 +++++++------ 4 files changed, 109 insertions(+), 53 deletions(-) create mode 100644 .chloggen/container_parser_resource_attributes.yaml diff --git a/.chloggen/container_parser_resource_attributes.yaml b/.chloggen/container_parser_resource_attributes.yaml new file mode 100644 index 000000000000..ac0d3ec8e015 --- /dev/null +++ b/.chloggen/container_parser_resource_attributes.yaml @@ -0,0 +1,28 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: bug_fix + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: filelogreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: "Container parser should add k8s metadata as resource attributes and not as log record attributes" + + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [33341] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/pkg/stanza/docs/operators/container.md b/pkg/stanza/docs/operators/container.md index 8c3fe381eccc..1930a73b17f7 100644 --- a/pkg/stanza/docs/operators/container.md +++ b/pkg/stanza/docs/operators/container.md @@ -31,16 +31,18 @@ will produce the following k8s metadata: ```json { - "attributes": { - "k8s": { - "container": { - "name": "kube-controller", - "restart_count": "1" - }, "pod": { - "uid": "49cc7c1fd3702c40b2686ea7486091d6", - "name": "kube-controller-kind-control-plane" - }, "namespace": { - "name": "some-ns" + "resource": { + "attributes": { + "k8s": { + "container": { + "name": "kube-controller", + "restart_count": "1" + }, "pod": { + "uid": "49cc7c1fd3702c40b2686ea7486091d6", + "name": "kube-controller-kind-control-plane" + }, "namespace": { + "name": "some-ns" + } } } } @@ -84,12 +86,16 @@ Note: in this example the `format: docker` is optional since formats can be auto "attributes": { "time": "2024-03-30T08:31:20.545192187Z", "log.iostream": "stdout", - "k8s.pod.name": "kube-controller-kind-control-plane", - "k8s.pod.uid": "49cc7c1fd3702c40b2686ea7486091d6", - "k8s.container.name": "kube-controller", - "k8s.container.restart_count": "1", - "k8s.namespace.name": "some", "log.file.path": "/var/log/pods/some_kube-controller-kind-control-plane_49cc7c1fd3702c40b2686ea7486091d6/kube-controller/1.log" + }, + "resource": { + "attributes": { + "k8s.pod.name": "kube-controller-kind-control-plane", + "k8s.pod.uid": "49cc7c1fd3702c40b2686ea7486091d6", + "k8s.container.name": "kube-controller", + "k8s.container.restart_count": "1", + "k8s.namespace.name": "some" + } } } ``` @@ -129,12 +135,16 @@ Configuration: "time": "2024-04-13T07:59:37.505201169-05:00", "logtag": "F", "log.iostream": "stdout", - "k8s.pod.name": "kube-controller-kind-control-plane", - "k8s.pod.uid": "49cc7c1fd3702c40b2686ea7486091d6", - "k8s.container.name": "kube-controller", - "k8s.container.restart_count": "1", - "k8s.namespace.name": "some", "log.file.path": "/var/log/pods/some_kube-controller-kind-control-plane_49cc7c1fd3702c40b2686ea7486091d6/kube-controller/1.log" + }, + "resource": { + "attributes": { + "k8s.pod.name": "kube-controller-kind-control-plane", + "k8s.pod.uid": "49cc7c1fd3702c40b2686ea7486091d6", + "k8s.container.name": "kube-controller", + "k8s.container.restart_count": "1", + "k8s.namespace.name": "some" + } } } ``` @@ -174,12 +184,16 @@ Configuration: "time": "2023-06-22T10:27:25.813799277Z", "logtag": "F", "log.iostream": "stdout", - "k8s.pod.name": "kube-controller-kind-control-plane", - "k8s.pod.uid": "49cc7c1fd3702c40b2686ea7486091d6", - "k8s.container.name": "kube-controller", - "k8s.container.restart_count": "1", - "k8s.namespace.name": "some", "log.file.path": "/var/log/pods/some_kube-controller-kind-control-plane_49cc7c1fd3702c40b2686ea7486091d6/kube-controller/1.log" + }, + "resource": { + "attributes": { + "k8s.pod.name": "kube-controller-kind-control-plane", + "k8s.pod.uid": "49cc7c1fd3702c40b2686ea7486091d6", + "k8s.container.name": "kube-controller", + "k8s.container.restart_count": "1", + "k8s.namespace.name": "some" + } } } ``` @@ -227,12 +241,16 @@ Configuration: "time": "2023-06-22T10:27:25.813799277Z", "logtag": "F", "log.iostream": "stdout", - "k8s.pod.name": "kube-controller-kind-control-plane", - "k8s.pod.uid": "49cc7c1fd3702c40b2686ea7486091d6", - "k8s.container.name": "kube-controller", - "k8s.container.restart_count": "1", - "k8s.namespace.name": "some", "log.file.path": "/var/log/pods/some_kube-controller-kind-control-plane_49cc7c1fd3702c40b2686ea7486091d6/kube-controller/1.log" + }, + "resource": { + "attributes": { + "k8s.pod.name": "kube-controller-kind-control-plane", + "k8s.pod.uid": "49cc7c1fd3702c40b2686ea7486091d6", + "k8s.container.name": "kube-controller", + "k8s.container.restart_count": "1", + "k8s.namespace.name": "some" + } } } ``` diff --git a/pkg/stanza/operator/parser/container/parser.go b/pkg/stanza/operator/parser/container/parser.go index 384097c53283..3c09ee2ada4f 100644 --- a/pkg/stanza/operator/parser/container/parser.go +++ b/pkg/stanza/operator/parser/container/parser.go @@ -300,7 +300,7 @@ func (p *Parser) extractk8sMetaFromFilePath(e *entry.Entry) error { } for originalKey, attributeKey := range k8sMetadataMapping { - newField := entry.NewAttributeField(attributeKey) + newField := entry.NewResourceField(attributeKey) if err := newField.Set(e, parsedValues[originalKey]); err != nil { return fmt.Errorf("failed to set %v as metadata at %v", originalKey, attributeKey) } diff --git a/pkg/stanza/operator/parser/container/parser_test.go b/pkg/stanza/operator/parser/container/parser_test.go index 71e159daacb2..924f83546f4f 100644 --- a/pkg/stanza/operator/parser/container/parser_test.go +++ b/pkg/stanza/operator/parser/container/parser_test.go @@ -157,16 +157,18 @@ func TestProcess(t *testing.T) { }, &entry.Entry{ Attributes: map[string]any{ - "time": "2029-03-30T08:31:20.545192187Z", - "log.iostream": "stdout", + "time": "2029-03-30T08:31:20.545192187Z", + "log.iostream": "stdout", + "log.file.path": "/var/log/pods/some_kube-scheduler-kind-control-plane_49cc7c1fd3702c40b2686ea7486091d3/kube-scheduler44/1.log", + }, + Body: "INFO: log line here", + Resource: map[string]any{ "k8s.pod.name": "kube-scheduler-kind-control-plane", "k8s.pod.uid": "49cc7c1fd3702c40b2686ea7486091d3", "k8s.container.name": "kube-scheduler44", "k8s.container.restart_count": "1", "k8s.namespace.name": "some", - "log.file.path": "/var/log/pods/some_kube-scheduler-kind-control-plane_49cc7c1fd3702c40b2686ea7486091d3/kube-scheduler44/1.log", }, - Body: "INFO: log line here", Timestamp: time.Date(2029, time.March, 30, 8, 31, 20, 545192187, time.UTC), }, }, @@ -212,17 +214,19 @@ func TestRecombineProcess(t *testing.T) { []*entry.Entry{ { Attributes: map[string]any{ - "time": "2024-04-13T07:59:37.505201169-10:00", - "log.iostream": "stdout", - "logtag": "F", + "time": "2024-04-13T07:59:37.505201169-10:00", + "log.iostream": "stdout", + "logtag": "F", + "log.file.path": "/var/log/pods/some_kube-scheduler-kind-control-plane_49cc7c1fd3702c40b2686ea7486091d3/kube-scheduler44/1.log", + }, + Body: "standalone crio line which is awesome!", + Resource: map[string]any{ "k8s.pod.name": "kube-scheduler-kind-control-plane", "k8s.pod.uid": "49cc7c1fd3702c40b2686ea7486091d3", "k8s.container.name": "kube-scheduler44", "k8s.container.restart_count": "1", "k8s.namespace.name": "some", - "log.file.path": "/var/log/pods/some_kube-scheduler-kind-control-plane_49cc7c1fd3702c40b2686ea7486091d3/kube-scheduler44/1.log", }, - Body: "standalone crio line which is awesome!", Timestamp: time.Date(2024, time.April, 13, 7, 59, 37, 505201169, time.FixedZone("", -10*60*60)), }, }, @@ -246,15 +250,17 @@ func TestRecombineProcess(t *testing.T) { []*entry.Entry{ { Attributes: map[string]any{ - "time": "2024-04-13T07:59:37.505201169Z", - "log.iostream": "stdout", - "logtag": "F", + "time": "2024-04-13T07:59:37.505201169Z", + "log.iostream": "stdout", + "logtag": "F", + "log.file.path": "/var/log/pods/some_kube-scheduler-kind-control-plane_49cc7c1fd3702c40b2686ea7486091d3/kube-scheduler44/1.log", + }, + Resource: map[string]any{ "k8s.pod.name": "kube-scheduler-kind-control-plane", "k8s.pod.uid": "49cc7c1fd3702c40b2686ea7486091d3", "k8s.container.name": "kube-scheduler44", "k8s.container.restart_count": "1", "k8s.namespace.name": "some", - "log.file.path": "/var/log/pods/some_kube-scheduler-kind-control-plane_49cc7c1fd3702c40b2686ea7486091d3/kube-scheduler44/1.log", }, Body: "standalone containerd line which is awesome!", Timestamp: time.Date(2024, time.April, 13, 7, 59, 37, 505201169, time.UTC), @@ -286,15 +292,17 @@ func TestRecombineProcess(t *testing.T) { []*entry.Entry{ { Attributes: map[string]any{ - "time": "2024-04-13T07:59:37.505201169-10:00", - "log.iostream": "stdout", - "logtag": "P", + "time": "2024-04-13T07:59:37.505201169-10:00", + "log.iostream": "stdout", + "logtag": "P", + "log.file.path": "/var/log/pods/some_kube-scheduler-kind-control-plane_49cc7c1fd3702c40b2686ea7486091d3/kube-scheduler44/1.log", + }, + Resource: map[string]any{ "k8s.pod.name": "kube-scheduler-kind-control-plane", "k8s.pod.uid": "49cc7c1fd3702c40b2686ea7486091d3", "k8s.container.name": "kube-scheduler44", "k8s.container.restart_count": "1", "k8s.namespace.name": "some", - "log.file.path": "/var/log/pods/some_kube-scheduler-kind-control-plane_49cc7c1fd3702c40b2686ea7486091d3/kube-scheduler44/1.log", }, Body: "standalone crio line which is awesome!", Timestamp: time.Date(2024, time.April, 13, 7, 59, 37, 505201169, time.FixedZone("", -10*60*60)), @@ -326,17 +334,19 @@ func TestRecombineProcess(t *testing.T) { []*entry.Entry{ { Attributes: map[string]any{ - "time": "2024-04-13T07:59:37.505201169Z", - "log.iostream": "stdout", - "logtag": "P", + "time": "2024-04-13T07:59:37.505201169Z", + "log.iostream": "stdout", + "logtag": "P", + "log.file.path": "/var/log/pods/some_kube-scheduler-kind-control-plane_49cc7c1fd3702c40b2686ea7486091d3/kube-scheduler44/1.log", + }, + Body: "standalone containerd line which is awesome!", + Resource: map[string]any{ "k8s.pod.name": "kube-scheduler-kind-control-plane", "k8s.pod.uid": "49cc7c1fd3702c40b2686ea7486091d3", "k8s.container.name": "kube-scheduler44", "k8s.container.restart_count": "1", "k8s.namespace.name": "some", - "log.file.path": "/var/log/pods/some_kube-scheduler-kind-control-plane_49cc7c1fd3702c40b2686ea7486091d3/kube-scheduler44/1.log", }, - Body: "standalone containerd line which is awesome!", Timestamp: time.Date(2024, time.April, 13, 7, 59, 37, 505201169, time.UTC), }, }, From c33786773f9945c1ed42adee41bf5c22a129835e Mon Sep 17 00:00:00 2001 From: ChrsMark Date: Tue, 4 Jun 2024 17:11:30 +0300 Subject: [PATCH 2/2] fix wrong doc Signed-off-by: ChrsMark --- pkg/stanza/docs/operators/container.md | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/pkg/stanza/docs/operators/container.md b/pkg/stanza/docs/operators/container.md index 1930a73b17f7..07895a955fc0 100644 --- a/pkg/stanza/docs/operators/container.md +++ b/pkg/stanza/docs/operators/container.md @@ -33,17 +33,11 @@ will produce the following k8s metadata: { "resource": { "attributes": { - "k8s": { - "container": { - "name": "kube-controller", - "restart_count": "1" - }, "pod": { - "uid": "49cc7c1fd3702c40b2686ea7486091d6", - "name": "kube-controller-kind-control-plane" - }, "namespace": { - "name": "some-ns" - } - } + "k8s.pod.name": "kube-controller-kind-control-plane", + "k8s.pod.uid": "49cc7c1fd3702c40b2686ea7486091d6", + "k8s.container.name": "kube-controller", + "k8s.container.restart_count": "1", + "k8s.namespace.name": "some" } } }