From 2ecf2ae317c2ebbe4b19349e83386255e9170af0 Mon Sep 17 00:00:00 2001 From: Brian Floersch Date: Sun, 4 Jun 2023 08:45:26 -0400 Subject: [PATCH] fix auto multi-line integration config (#17447) * fix auto multi-line integration config * reno * update tests --- .../launchers/container/tailerfactory/file.go | 34 +++++++++++-------- .../container/tailerfactory/file_test.go | 33 ++++++++++++------ ...e-integration-config-35449268a4471c19.yaml | 13 +++++++ 3 files changed, 56 insertions(+), 24 deletions(-) create mode 100644 releasenotes/notes/fix-auto-multi-line-integration-config-35449268a4471c19.yaml diff --git a/pkg/logs/internal/launchers/container/tailerfactory/file.go b/pkg/logs/internal/launchers/container/tailerfactory/file.go index d09b1a584bd2d1..1d996f08cb3bbf 100644 --- a/pkg/logs/internal/launchers/container/tailerfactory/file.go +++ b/pkg/logs/internal/launchers/container/tailerfactory/file.go @@ -120,13 +120,16 @@ func (tf *factory) makeDockerFileSource(source *sources.LogSource) (*sources.Log // New file source that inherits most of its parent's properties fileSource := sources.NewLogSource(source.Name, &config.LogsConfig{ - Type: config.FileType, - Identifier: containerID, - Path: path, - Service: serviceName, - Source: sourceName, - Tags: source.Config.Tags, - ProcessingRules: source.Config.ProcessingRules, + Type: config.FileType, + Identifier: containerID, + Path: path, + Service: serviceName, + Source: sourceName, + Tags: source.Config.Tags, + ProcessingRules: source.Config.ProcessingRules, + AutoMultiLine: source.Config.AutoMultiLine, + AutoMultiLineSampleSize: source.Config.AutoMultiLineSampleSize, + AutoMultiLineMatchThreshold: source.Config.AutoMultiLineMatchThreshold, }) // inform the file launcher that it should expect docker-formatted content @@ -203,13 +206,16 @@ func (tf *factory) makeK8sFileSource(source *sources.LogSource) (*sources.LogSou fileSource := sources.NewLogSource( fmt.Sprintf("%s/%s/%s", pod.Namespace, pod.Name, container.Name), &config.LogsConfig{ - Type: config.FileType, - Identifier: containerID, - Path: path, - Service: serviceName, - Source: sourceName, - Tags: source.Config.Tags, - ProcessingRules: source.Config.ProcessingRules, + Type: config.FileType, + Identifier: containerID, + Path: path, + Service: serviceName, + Source: sourceName, + Tags: source.Config.Tags, + ProcessingRules: source.Config.ProcessingRules, + AutoMultiLine: source.Config.AutoMultiLine, + AutoMultiLineSampleSize: source.Config.AutoMultiLineSampleSize, + AutoMultiLineMatchThreshold: source.Config.AutoMultiLineMatchThreshold, }) switch source.Config.Type { diff --git a/pkg/logs/internal/launchers/container/tailerfactory/file_test.go b/pkg/logs/internal/launchers/container/tailerfactory/file_test.go index 780ee10aca9c10..81d5c9e296765e 100644 --- a/pkg/logs/internal/launchers/container/tailerfactory/file_test.go +++ b/pkg/logs/internal/launchers/container/tailerfactory/file_test.go @@ -22,6 +22,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/logs/pipeline" "github.com/DataDog/datadog-agent/pkg/logs/sources" dockerutilPkg "github.com/DataDog/datadog-agent/pkg/util/docker" + "github.com/DataDog/datadog-agent/pkg/util/pointer" "github.com/DataDog/datadog-agent/pkg/workloadmeta" ) @@ -85,11 +86,14 @@ func TestMakeFileSource_docker_success(t *testing.T) { cop: containersorpods.NewDecidedChooser(containersorpods.LogContainers), } source := sources.NewLogSource("test", &config.LogsConfig{ - Type: "docker", - Identifier: "abc", - Source: "src", - Service: "svc", - Tags: []string{"tag!"}, + Type: "docker", + Identifier: "abc", + Source: "src", + Service: "svc", + Tags: []string{"tag!"}, + AutoMultiLine: pointer.Ptr(true), + AutoMultiLineSampleSize: 123, + AutoMultiLineMatchThreshold: 0.123, }) child, err := tf.makeFileSource(source) require.NoError(t, err) @@ -101,6 +105,9 @@ func TestMakeFileSource_docker_success(t *testing.T) { require.Equal(t, source.Config.Service, child.Config.Service) require.Equal(t, source.Config.Tags, child.Config.Tags) require.Equal(t, sources.DockerSourceType, child.GetSourceType()) + require.Equal(t, *source.Config.AutoMultiLine, true) + require.Equal(t, source.Config.AutoMultiLineSampleSize, 123) + require.Equal(t, source.Config.AutoMultiLineMatchThreshold, 0.123) } func TestMakeFileSource_docker_no_file(t *testing.T) { @@ -179,11 +186,14 @@ func TestMakeK8sSource(t *testing.T) { for _, sourceConfigType := range []string{"docker", "containerd"} { t.Run("source.Config.Type="+sourceConfigType, func(t *testing.T) { source := sources.NewLogSource("test", &config.LogsConfig{ - Type: sourceConfigType, - Identifier: "abc", - Source: "src", - Service: "svc", - Tags: []string{"tag!"}, + Type: sourceConfigType, + Identifier: "abc", + Source: "src", + Service: "svc", + Tags: []string{"tag!"}, + AutoMultiLine: pointer.Ptr(true), + AutoMultiLineSampleSize: 123, + AutoMultiLineMatchThreshold: 0.123, }) child, err := tf.makeK8sFileSource(source) require.NoError(t, err) @@ -194,6 +204,9 @@ func TestMakeK8sSource(t *testing.T) { require.Equal(t, "src", child.Config.Source) require.Equal(t, "svc", child.Config.Service) require.Equal(t, []string{"tag!"}, child.Config.Tags) + require.Equal(t, *child.Config.AutoMultiLine, true) + require.Equal(t, child.Config.AutoMultiLineSampleSize, 123) + require.Equal(t, child.Config.AutoMultiLineMatchThreshold, 0.123) switch sourceConfigType { case "docker": require.Equal(t, sources.DockerSourceType, child.GetSourceType()) diff --git a/releasenotes/notes/fix-auto-multi-line-integration-config-35449268a4471c19.yaml b/releasenotes/notes/fix-auto-multi-line-integration-config-35449268a4471c19.yaml new file mode 100644 index 00000000000000..1269b850afabba --- /dev/null +++ b/releasenotes/notes/fix-auto-multi-line-integration-config-35449268a4471c19.yaml @@ -0,0 +1,13 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +fixes: + - | + Fix an issue where ``auto_multi_line_detection``, ``auto_multi_line_sample_size``, + and ``auto_multi_line_match_threshold`` were not working when set though a pod + annotation or container label. \ No newline at end of file