From f1671b6b456069381266241cde4212263356f3ff Mon Sep 17 00:00:00 2001 From: Dan Jaglowski Date: Tue, 3 Dec 2024 10:51:31 -0500 Subject: [PATCH] Add golden test cases --- processor/unrollprocessor/config.go | 3 +- processor/unrollprocessor/go.mod | 4 + processor/unrollprocessor/go.sum | 8 ++ processor/unrollprocessor/processor.go | 16 ++- processor/unrollprocessor/processor_test.go | 57 +++++++++ .../testdata/mixed_slice_types/expected.yaml | 73 +++++++++++ .../testdata/mixed_slice_types/input.yaml | 57 +++++++++ .../testdata/nop/expected.yaml | 16 +++ .../unrollprocessor/testdata/nop/input.yaml | 16 +++ .../testdata/recursive_false/expected.yaml | 22 ++++ .../testdata/recursive_false/input.yaml | 19 +++ .../testdata/recursive_true/expected.yaml | 29 +++++ .../testdata/recursive_true/input.yaml | 19 +++ .../testdata/simple/expected.yaml | 121 ++++++++++++++++++ .../testdata/simple/input.yaml | 97 ++++++++++++++ .../testdata/some_not_slices/expected.yaml | 34 +++++ .../testdata/some_not_slices/input.yaml | 28 ++++ 17 files changed, 617 insertions(+), 2 deletions(-) create mode 100644 processor/unrollprocessor/testdata/mixed_slice_types/expected.yaml create mode 100644 processor/unrollprocessor/testdata/mixed_slice_types/input.yaml create mode 100644 processor/unrollprocessor/testdata/nop/expected.yaml create mode 100644 processor/unrollprocessor/testdata/nop/input.yaml create mode 100644 processor/unrollprocessor/testdata/recursive_false/expected.yaml create mode 100644 processor/unrollprocessor/testdata/recursive_false/input.yaml create mode 100644 processor/unrollprocessor/testdata/recursive_true/expected.yaml create mode 100644 processor/unrollprocessor/testdata/recursive_true/input.yaml create mode 100644 processor/unrollprocessor/testdata/simple/expected.yaml create mode 100644 processor/unrollprocessor/testdata/simple/input.yaml create mode 100644 processor/unrollprocessor/testdata/some_not_slices/expected.yaml create mode 100644 processor/unrollprocessor/testdata/some_not_slices/input.yaml diff --git a/processor/unrollprocessor/config.go b/processor/unrollprocessor/config.go index 9bd1caa68..c0c5199b6 100644 --- a/processor/unrollprocessor/config.go +++ b/processor/unrollprocessor/config.go @@ -21,7 +21,8 @@ import ( // Config is the configuration for the unroll processor. type Config struct { - Field UnrollField `mapstructure:"field"` + Field UnrollField `mapstructure:"field"` + Recursive bool `mapstructure:"recursive"` } // UnrollField is the field to unroll. diff --git a/processor/unrollprocessor/go.mod b/processor/unrollprocessor/go.mod index 641a7a352..d9996d89f 100644 --- a/processor/unrollprocessor/go.mod +++ b/processor/unrollprocessor/go.mod @@ -3,6 +3,8 @@ module github.com/observiq/bindplane-agent/processor/unrollprocessor go 1.23.3 require ( + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.114.0 + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.114.0 github.com/stretchr/testify v1.9.0 go.opentelemetry.io/collector/component v0.114.0 go.opentelemetry.io/collector/component/componenttest v0.114.0 @@ -16,6 +18,7 @@ require ( ) require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -30,6 +33,7 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.114.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect go.opentelemetry.io/collector/component/componentstatus v0.114.0 // indirect diff --git a/processor/unrollprocessor/go.sum b/processor/unrollprocessor/go.sum index 2c0a61c2b..aaca7d2f8 100644 --- a/processor/unrollprocessor/go.sum +++ b/processor/unrollprocessor/go.sum @@ -1,3 +1,5 @@ +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -38,6 +40,12 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.114.0 h1:SXi6JSSs2cWROnC1U2v3XysG3t58ilGUwoLqxpGuwFU= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.114.0/go.mod h1:LSd6sus2Jvpg3M3vM4HgmVh3/dmMtcJmTqELrFOQFRg= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.114.0 h1:m8uPYU2rTj0sKiYgzCvIPajD3meiYsu+nX0hplUnlEU= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.114.0/go.mod h1:P0BaP92pXPkTyTmObfLYUoRBfMYU+i0hdS3oM1DpGJo= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.114.0 h1:Qg80zPfNMlub7LO07VMDElOu3M2oxqdZgvvB+X72a4U= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.114.0/go.mod h1:5qsGcjFV3WFI6J2onAlkR7Xd/8VtwJcECaDRZfW4Tb4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= diff --git a/processor/unrollprocessor/processor.go b/processor/unrollprocessor/processor.go index d9b25618c..1a6f59abc 100644 --- a/processor/unrollprocessor/processor.go +++ b/processor/unrollprocessor/processor.go @@ -30,7 +30,14 @@ func (p *unrollProcessor) ProcessLogs(_ context.Context, ld plog.Logs) (plog.Log rls := ld.ResourceLogs().At(i) for j := 0; j < rls.ScopeLogs().Len(); j++ { sls := rls.ScopeLogs().At(j) - for k := 0; k < sls.LogRecords().Len(); k++ { + origLen := sls.LogRecords().Len() + var last func() int + if p.cfg.Recursive { + last = sls.LogRecords().Len + } else { + last = func() int { return origLen } + } + for k := 0; k < last(); k++ { lr := sls.LogRecords().At(k) if lr.Body().Type() != pcommon.ValueTypeSlice { continue @@ -42,6 +49,13 @@ func (p *unrollProcessor) ProcessLogs(_ context.Context, ld plog.Logs) (plog.Log } } sls.LogRecords().RemoveIf(func(lr plog.LogRecord) bool { + if p.cfg.Recursive { + return lr.Body().Type() == pcommon.ValueTypeSlice + } + if origLen == 0 { + return false + } + origLen-- return lr.Body().Type() == pcommon.ValueTypeSlice }) } diff --git a/processor/unrollprocessor/processor_test.go b/processor/unrollprocessor/processor_test.go index 69e908a6a..f3eb60067 100644 --- a/processor/unrollprocessor/processor_test.go +++ b/processor/unrollprocessor/processor_test.go @@ -2,9 +2,16 @@ package unrollprocessor import ( "context" + "path/filepath" "testing" + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden" + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest/plogtest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/consumer/consumertest" "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/processor/processortest" ) func BenchmarkUnroll(b *testing.B) { @@ -29,3 +36,53 @@ func createTestResourceLogs() plog.Logs { } return rl } + +func TestProcessor(t *testing.T) { + for _, test := range []struct { + name string + recursive bool + }{ + { + name: "nop", + }, + { + name: "simple", + }, + { + name: "mixed_slice_types", + }, + { + name: "some_not_slices", + }, + { + name: "recursive_false", + }, + { + name: "recursive_true", + recursive: true, + }, + } { + t.Run(test.name, func(t *testing.T) { + input, err := golden.ReadLogs(filepath.Join("testdata", test.name, "input.yaml")) + require.NoError(t, err) + expected, err := golden.ReadLogs(filepath.Join("testdata", test.name, "expected.yaml")) + require.NoError(t, err) + + f := NewFactory() + cfg := f.CreateDefaultConfig().(*Config) + cfg.Recursive = test.recursive + set := processortest.NewNopSettings() + sink := &consumertest.LogsSink{} + p, err := f.CreateLogs(context.Background(), set, cfg, sink) + require.NoError(t, err) + + err = p.ConsumeLogs(context.Background(), input) + require.NoError(t, err) + + actual := sink.AllLogs() + require.Equal(t, 1, len(actual)) + + assert.NoError(t, plogtest.CompareLogs(expected, actual[0])) + }) + } +} diff --git a/processor/unrollprocessor/testdata/mixed_slice_types/expected.yaml b/processor/unrollprocessor/testdata/mixed_slice_types/expected.yaml new file mode 100644 index 000000000..a111b370f --- /dev/null +++ b/processor/unrollprocessor/testdata/mixed_slice_types/expected.yaml @@ -0,0 +1,73 @@ +resourceLogs: + - resource: + scopeLogs: + - logRecords: + - attributes: + - key: recordName + value: + stringValue: strings + body: + stringValue: value1 + - attributes: + - key: recordName + value: + stringValue: strings + body: + stringValue: value2 + - attributes: + - key: recordName + value: + stringValue: ints + body: + intValue: 1 + - attributes: + - key: recordName + value: + stringValue: ints + body: + intValue: 2 + - attributes: + - key: recordName + value: + stringValue: doubles + body: + doubleValue: 1.1 + - attributes: + - key: recordName + value: + stringValue: doubles + body: + doubleValue: 2.2 + - attributes: + - key: recordName + value: + stringValue: bools + body: + boolValue: true + - attributes: + - key: recordName + value: + stringValue: bools + body: + boolValue: false + - attributes: + - key: recordName + value: + stringValue: maps + body: + kvlistValue: + values: + - key: foo + value: + stringValue: bar + - attributes: + - key: recordName + value: + stringValue: maps + body: + kvlistValue: + values: + - key: hello + value: + stringValue: world + diff --git a/processor/unrollprocessor/testdata/mixed_slice_types/input.yaml b/processor/unrollprocessor/testdata/mixed_slice_types/input.yaml new file mode 100644 index 000000000..26c1823aa --- /dev/null +++ b/processor/unrollprocessor/testdata/mixed_slice_types/input.yaml @@ -0,0 +1,57 @@ +resourceLogs: + - resource: + scopeLogs: + - logRecords: + - attributes: + - key: recordName + value: + stringValue: strings + body: + arrayValue: + values: + - stringValue: value1 + - stringValue: value2 + - attributes: + - key: recordName + value: + stringValue: ints + body: + arrayValue: + values: + - intValue: 1 + - intValue: 2 + - attributes: + - key: recordName + value: + stringValue: doubles + body: + arrayValue: + values: + - doubleValue: 1.1 + - doubleValue: 2.2 + - attributes: + - key: recordName + value: + stringValue: bools + body: + arrayValue: + values: + - boolValue: true + - boolValue: false + - attributes: + - key: recordName + value: + stringValue: maps + body: + arrayValue: + values: + - kvlistValue: + values: + - key: foo + value: + stringValue: bar + - kvlistValue: + values: + - key: hello + value: + stringValue: world diff --git a/processor/unrollprocessor/testdata/nop/expected.yaml b/processor/unrollprocessor/testdata/nop/expected.yaml new file mode 100644 index 000000000..79873927f --- /dev/null +++ b/processor/unrollprocessor/testdata/nop/expected.yaml @@ -0,0 +1,16 @@ +resourceLogs: + - resource: + scopeLogs: + - logRecords: + - attributes: + - key: recordName + value: + stringValue: recordA + body: + stringValue: value1 + - attributes: + - key: recordName + value: + stringValue: recordB + body: + stringValue: value1 \ No newline at end of file diff --git a/processor/unrollprocessor/testdata/nop/input.yaml b/processor/unrollprocessor/testdata/nop/input.yaml new file mode 100644 index 000000000..79873927f --- /dev/null +++ b/processor/unrollprocessor/testdata/nop/input.yaml @@ -0,0 +1,16 @@ +resourceLogs: + - resource: + scopeLogs: + - logRecords: + - attributes: + - key: recordName + value: + stringValue: recordA + body: + stringValue: value1 + - attributes: + - key: recordName + value: + stringValue: recordB + body: + stringValue: value1 \ No newline at end of file diff --git a/processor/unrollprocessor/testdata/recursive_false/expected.yaml b/processor/unrollprocessor/testdata/recursive_false/expected.yaml new file mode 100644 index 000000000..42243198f --- /dev/null +++ b/processor/unrollprocessor/testdata/recursive_false/expected.yaml @@ -0,0 +1,22 @@ +resourceLogs: + - resource: + scopeLogs: + - logRecords: + - attributes: + - key: recordName + value: + stringValue: slices + body: + arrayValue: + values: + - stringValue: one + - stringValue: two + - attributes: + - key: recordName + value: + stringValue: slices + body: + arrayValue: + values: + - stringValue: three + - stringValue: four diff --git a/processor/unrollprocessor/testdata/recursive_false/input.yaml b/processor/unrollprocessor/testdata/recursive_false/input.yaml new file mode 100644 index 000000000..f7b018b5f --- /dev/null +++ b/processor/unrollprocessor/testdata/recursive_false/input.yaml @@ -0,0 +1,19 @@ +resourceLogs: + - resource: + scopeLogs: + - logRecords: + - attributes: + - key: recordName + value: + stringValue: slices + body: + arrayValue: + values: + - arrayValue: + values: + - stringValue: one + - stringValue: two + - arrayValue: + values: + - stringValue: three + - stringValue: four diff --git a/processor/unrollprocessor/testdata/recursive_true/expected.yaml b/processor/unrollprocessor/testdata/recursive_true/expected.yaml new file mode 100644 index 000000000..f8f9a6e5d --- /dev/null +++ b/processor/unrollprocessor/testdata/recursive_true/expected.yaml @@ -0,0 +1,29 @@ +resourceLogs: + - resource: + scopeLogs: + - logRecords: + - attributes: + - key: recordName + value: + stringValue: slices + body: + stringValue: one + - attributes: + - key: recordName + value: + stringValue: slices + body: + stringValue: two + - attributes: + - key: recordName + value: + stringValue: slices + body: + stringValue: three + - attributes: + - key: recordName + value: + stringValue: slices + body: + stringValue: four + diff --git a/processor/unrollprocessor/testdata/recursive_true/input.yaml b/processor/unrollprocessor/testdata/recursive_true/input.yaml new file mode 100644 index 000000000..f7b018b5f --- /dev/null +++ b/processor/unrollprocessor/testdata/recursive_true/input.yaml @@ -0,0 +1,19 @@ +resourceLogs: + - resource: + scopeLogs: + - logRecords: + - attributes: + - key: recordName + value: + stringValue: slices + body: + arrayValue: + values: + - arrayValue: + values: + - stringValue: one + - stringValue: two + - arrayValue: + values: + - stringValue: three + - stringValue: four diff --git a/processor/unrollprocessor/testdata/simple/expected.yaml b/processor/unrollprocessor/testdata/simple/expected.yaml new file mode 100644 index 000000000..418a08d3e --- /dev/null +++ b/processor/unrollprocessor/testdata/simple/expected.yaml @@ -0,0 +1,121 @@ +resourceLogs: + - resource: + attributes: + - key: resourceName + value: + stringValue: resourceA + scopeLogs: + - scope: + name: scopeA + logRecords: + - attributes: + - key: recordName + value: + stringValue: recordA + body: + stringValue: value1 + - attributes: + - key: recordName + value: + stringValue: recordA + body: + stringValue: value2 + - attributes: + - key: recordName + value: + stringValue: recordB + body: + stringValue: value1 + - attributes: + - key: recordName + value: + stringValue: recordB + body: + stringValue: value2 + - scope: + name: scopeB + logRecords: + - attributes: + - key: recordName + value: + stringValue: recordA + body: + stringValue: value1 + - attributes: + - key: recordName + value: + stringValue: recordA + body: + stringValue: value2 + - attributes: + - key: recordName + value: + stringValue: recordB + body: + stringValue: value1 + - attributes: + - key: recordName + value: + stringValue: recordB + body: + stringValue: value2 + - resource: + attributes: + - key: resourceName + value: + stringValue: resourceA + scopeLogs: + - scope: + name: scopeB + logRecords: + - attributes: + - key: recordName + value: + stringValue: recordA + body: + stringValue: value1 + - attributes: + - key: recordName + value: + stringValue: recordA + body: + stringValue: value2 + - attributes: + - key: recordName + value: + stringValue: recordB + body: + stringValue: value1 + - attributes: + - key: recordName + value: + stringValue: recordB + body: + stringValue: value2 + - scope: + name: scopeB + logRecords: + - attributes: + - key: recordName + value: + stringValue: recordA + body: + stringValue: value1 + - attributes: + - key: recordName + value: + stringValue: recordA + body: + stringValue: value2 + - attributes: + - key: recordName + value: + stringValue: recordB + body: + stringValue: value1 + - attributes: + - key: recordName + value: + stringValue: recordB + body: + stringValue: value2 diff --git a/processor/unrollprocessor/testdata/simple/input.yaml b/processor/unrollprocessor/testdata/simple/input.yaml new file mode 100644 index 000000000..c5ca99eaa --- /dev/null +++ b/processor/unrollprocessor/testdata/simple/input.yaml @@ -0,0 +1,97 @@ +resourceLogs: + - resource: + attributes: + - key: resourceName + value: + stringValue: resourceA + scopeLogs: + - scope: + name: scopeA + logRecords: + - attributes: + - key: recordName + value: + stringValue: recordA + body: + arrayValue: + values: + - stringValue: value1 + - stringValue: value2 + - attributes: + - key: recordName + value: + stringValue: recordB + body: + arrayValue: + values: + - stringValue: value1 + - stringValue: value2 + - scope: + name: scopeB + logRecords: + - attributes: + - key: recordName + value: + stringValue: recordA + body: + arrayValue: + values: + - stringValue: value1 + - stringValue: value2 + - attributes: + - key: recordName + value: + stringValue: recordB + body: + arrayValue: + values: + - stringValue: value1 + - stringValue: value2 + - resource: + attributes: + - key: resourceName + value: + stringValue: resourceA + scopeLogs: + - scope: + name: scopeB + logRecords: + - attributes: + - key: recordName + value: + stringValue: recordA + body: + arrayValue: + values: + - stringValue: value1 + - stringValue: value2 + - attributes: + - key: recordName + value: + stringValue: recordB + body: + arrayValue: + values: + - stringValue: value1 + - stringValue: value2 + - scope: + name: scopeB + logRecords: + - attributes: + - key: recordName + value: + stringValue: recordA + body: + arrayValue: + values: + - stringValue: value1 + - stringValue: value2 + - attributes: + - key: recordName + value: + stringValue: recordB + body: + arrayValue: + values: + - stringValue: value1 + - stringValue: value2 diff --git a/processor/unrollprocessor/testdata/some_not_slices/expected.yaml b/processor/unrollprocessor/testdata/some_not_slices/expected.yaml new file mode 100644 index 000000000..583da7f2f --- /dev/null +++ b/processor/unrollprocessor/testdata/some_not_slices/expected.yaml @@ -0,0 +1,34 @@ +resourceLogs: + - resource: + scopeLogs: + - logRecords: + - attributes: + - key: recordName + value: + stringValue: recordB + body: + stringValue: value1 + - attributes: + - key: recordName + value: + stringValue: recordA + body: + stringValue: value1 + - attributes: + - key: recordName + value: + stringValue: recordA + body: + stringValue: value2 + - attributes: + - key: recordName + value: + stringValue: recordC + body: + stringValue: value1 + - attributes: + - key: recordName + value: + stringValue: recordC + body: + stringValue: value2 diff --git a/processor/unrollprocessor/testdata/some_not_slices/input.yaml b/processor/unrollprocessor/testdata/some_not_slices/input.yaml new file mode 100644 index 000000000..c7380d858 --- /dev/null +++ b/processor/unrollprocessor/testdata/some_not_slices/input.yaml @@ -0,0 +1,28 @@ +resourceLogs: + - resource: + scopeLogs: + - logRecords: + - attributes: + - key: recordName + value: + stringValue: recordA + body: + arrayValue: + values: + - stringValue: value1 + - stringValue: value2 + - attributes: + - key: recordName + value: + stringValue: recordB + body: + stringValue: value1 + - attributes: + - key: recordName + value: + stringValue: recordC + body: + arrayValue: + values: + - stringValue: value1 + - stringValue: value2