Feat/unroll processor tests

processor/unrollprocessor/config.go

@@ -21,7 +21,8 @@ import (
 
 // Config is the configuration for the unroll processor.
 type Config struct {
-	Field UnrollField `mapstructure:"field"`
+	Field     UnrollField `mapstructure:"field"`
+	Recursive bool        `mapstructure:"recursive"`
 }
 
 // UnrollField is the field to unroll.

processor/unrollprocessor/go.mod

@@ -3,6 +3,8 @@ module github.com/observiq/bindplane-agent/processor/unrollprocessor
 go 1.23.3
 
 require (
+	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.114.0
+	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.114.0
 	github.com/stretchr/testify v1.9.0
 	go.opentelemetry.io/collector/component v0.114.0
 	go.opentelemetry.io/collector/component/componenttest v0.114.0
@@ -16,6 +18,7 @@ require (
 )
 
 require (
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
@@ -30,6 +33,7 @@ require (
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.114.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	go.opentelemetry.io/collector/component/componentstatus v0.114.0 // indirect

processor/unrollprocessor/go.sum

@@ -1,3 +1,5 @@
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -38,6 +40,12 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.114.0 h1:SXi6JSSs2cWROnC1U2v3XysG3t58ilGUwoLqxpGuwFU=
+github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.114.0/go.mod h1:LSd6sus2Jvpg3M3vM4HgmVh3/dmMtcJmTqELrFOQFRg=
+github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.114.0 h1:m8uPYU2rTj0sKiYgzCvIPajD3meiYsu+nX0hplUnlEU=
+github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.114.0/go.mod h1:P0BaP92pXPkTyTmObfLYUoRBfMYU+i0hdS3oM1DpGJo=
+github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.114.0 h1:Qg80zPfNMlub7LO07VMDElOu3M2oxqdZgvvB+X72a4U=
+github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.114.0/go.mod h1:5qsGcjFV3WFI6J2onAlkR7Xd/8VtwJcECaDRZfW4Tb4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=

processor/unrollprocessor/processor.go

@@ -30,7 +30,14 @@ func (p *unrollProcessor) ProcessLogs(_ context.Context, ld plog.Logs) (plog.Log
 		rls := ld.ResourceLogs().At(i)
 		for j := 0; j < rls.ScopeLogs().Len(); j++ {
 			sls := rls.ScopeLogs().At(j)
-			for k := 0; k < sls.LogRecords().Len(); k++ {
+			origLen := sls.LogRecords().Len()
+			var last func() int
+			if p.cfg.Recursive {
+				last = sls.LogRecords().Len
+			} else {
+				last = func() int { return origLen }
+			}
+			for k := 0; k < last(); k++ {
 				lr := sls.LogRecords().At(k)
 				if lr.Body().Type() != pcommon.ValueTypeSlice {
 					continue
@@ -42,6 +49,13 @@ func (p *unrollProcessor) ProcessLogs(_ context.Context, ld plog.Logs) (plog.Log
 				}
 			}
 			sls.LogRecords().RemoveIf(func(lr plog.LogRecord) bool {
+				if p.cfg.Recursive {
+					return lr.Body().Type() == pcommon.ValueTypeSlice
+				}
+				if origLen == 0 {
+					return false
+				}
+				origLen--
 				return lr.Body().Type() == pcommon.ValueTypeSlice
 			})
 		}

processor/unrollprocessor/processor_test.go

@@ -2,9 +2,16 @@ package unrollprocessor
 
 import (
 	"context"
+	"path/filepath"
 	"testing"
 
+	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden"
+	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest/plogtest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/consumer/consumertest"
 	"go.opentelemetry.io/collector/pdata/plog"
+	"go.opentelemetry.io/collector/processor/processortest"
 )
 
 func BenchmarkUnroll(b *testing.B) {
@@ -29,3 +36,53 @@ func createTestResourceLogs() plog.Logs {
 	}
 	return rl
 }
+
+func TestProcessor(t *testing.T) {
+	for _, test := range []struct {
+		name      string
+		recursive bool
+	}{
+		{
+			name: "nop",
+		},
+		{
+			name: "simple",
+		},
+		{
+			name: "mixed_slice_types",
+		},
+		{
+			name: "some_not_slices",
+		},
+		{
+			name: "recursive_false",
+		},
+		{
+			name:      "recursive_true",
+			recursive: true,
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			input, err := golden.ReadLogs(filepath.Join("testdata", test.name, "input.yaml"))
+			require.NoError(t, err)
+			expected, err := golden.ReadLogs(filepath.Join("testdata", test.name, "expected.yaml"))
+			require.NoError(t, err)
+
+			f := NewFactory()
+			cfg := f.CreateDefaultConfig().(*Config)
+			cfg.Recursive = test.recursive
+			set := processortest.NewNopSettings()
+			sink := &consumertest.LogsSink{}
+			p, err := f.CreateLogs(context.Background(), set, cfg, sink)
+			require.NoError(t, err)
+
+			err = p.ConsumeLogs(context.Background(), input)
+			require.NoError(t, err)
+
+			actual := sink.AllLogs()
+			require.Equal(t, 1, len(actual))
+
+			assert.NoError(t, plogtest.CompareLogs(expected, actual[0]))
+		})
+	}
+}

processor/unrollprocessor/testdata/mixed_slice_types/expected.yaml

@@ -0,0 +1,73 @@
+resourceLogs:
+  - resource:
+    scopeLogs:
+      - logRecords:
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: strings
+            body:
+              stringValue: value1
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: strings
+            body:
+              stringValue: value2
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: ints
+            body:
+              intValue: 1
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: ints
+            body:
+              intValue: 2
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: doubles
+            body:
+              doubleValue: 1.1
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: doubles
+            body:
+              doubleValue: 2.2
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: bools
+            body:
+              boolValue: true
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: bools
+            body:
+              boolValue: false
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: maps
+            body:
+              kvlistValue:
+                values:
+                  - key: foo
+                    value:
+                      stringValue: bar
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: maps
+            body:
+              kvlistValue:
+                values:
+                  - key: hello
+                    value:
+                      stringValue: world
+

processor/unrollprocessor/testdata/mixed_slice_types/input.yaml

@@ -0,0 +1,57 @@
+resourceLogs:
+  - resource:
+    scopeLogs:
+      - logRecords:
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: strings
+            body:
+              arrayValue:
+                values:
+                  - stringValue: value1
+                  - stringValue: value2
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: ints
+            body:
+              arrayValue:
+                values:
+                  - intValue: 1
+                  - intValue: 2
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: doubles
+            body:
+              arrayValue:
+                values:
+                  - doubleValue: 1.1
+                  - doubleValue: 2.2
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: bools
+            body:
+              arrayValue:
+                values:
+                  - boolValue: true
+                  - boolValue: false
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: maps
+            body:
+              arrayValue:
+                values:
+                  - kvlistValue:
+                      values:
+                        - key: foo
+                          value:
+                            stringValue: bar
+                  - kvlistValue:
+                      values:
+                        - key: hello
+                          value:
+                            stringValue: world

processor/unrollprocessor/testdata/nop/expected.yaml

@@ -0,0 +1,16 @@
+resourceLogs:
+  - resource:
+    scopeLogs:
+      - logRecords:
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: recordA
+            body:
+              stringValue: value1
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: recordB
+            body:
+              stringValue: value1

processor/unrollprocessor/testdata/nop/input.yaml

@@ -0,0 +1,16 @@
+resourceLogs:
+  - resource:
+    scopeLogs:
+      - logRecords:
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: recordA
+            body:
+              stringValue: value1
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: recordB
+            body:
+              stringValue: value1

processor/unrollprocessor/testdata/recursive_false/expected.yaml

@@ -0,0 +1,22 @@
+resourceLogs:
+  - resource:
+    scopeLogs:
+      - logRecords:
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: slices
+            body:
+              arrayValue:
+                values:
+                  - stringValue: one
+                  - stringValue: two
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: slices
+            body:
+              arrayValue:
+                values:
+                  - stringValue: three
+                  - stringValue: four

processor/unrollprocessor/testdata/recursive_false/input.yaml

@@ -0,0 +1,19 @@
+resourceLogs:
+  - resource:
+    scopeLogs:
+      - logRecords:
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: slices
+            body:
+              arrayValue:
+                values:
+                  - arrayValue:
+                      values:
+                        - stringValue: one
+                        - stringValue: two
+                  - arrayValue:
+                      values:
+                        - stringValue: three
+                        - stringValue: four

processor/unrollprocessor/testdata/recursive_true/expected.yaml

@@ -0,0 +1,29 @@
+resourceLogs:
+  - resource:
+    scopeLogs:
+      - logRecords:
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: slices
+            body:
+              stringValue: one
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: slices
+            body:
+              stringValue: two
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: slices
+            body:
+              stringValue: three
+          - attributes:
+              - key: recordName
+                value:
+                  stringValue: slices
+            body:
+              stringValue: four
+