diff --git a/docs/processors.md b/docs/processors.md index aa0fb852a..823689112 100644 --- a/docs/processors.md +++ b/docs/processors.md @@ -16,6 +16,7 @@ Below is a list of supported processors with links to their documentation pages. | Log DeDuplication Processor | [logdeduplicationprocessor](../processor/logdeduplicationprocessor/README.md) | | Logs Transform Processor | [logstransform](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.108.0/processor/logstransformprocessor/README.md) | | Lookup Processor | [lookupprocessor](../processor/lookupprocessor/README.md) | +| Marshal Processor | [marshalprocessor](../processor/marshalprocessor/README.md) | | Mask Processor | [maskprocessor](../processor/maskprocessor/README.md) | | Memory Limiter Processor | [memorylimiterprocessor](https://github.com/open-telemetry/opentelemetry-collector/blob/v0.108.0/processor/memorylimiterprocessor/README.md) | | Metric Extract Processor | [metricextract](../processor/metricextractprocessor/README.md) | diff --git a/factories/processors.go b/factories/processors.go index 8a71bedda..ec0e00800 100644 --- a/factories/processors.go +++ b/factories/processors.go @@ -20,6 +20,7 @@ import ( "github.com/observiq/bindplane-agent/processor/logcountprocessor" "github.com/observiq/bindplane-agent/processor/logdeduplicationprocessor" "github.com/observiq/bindplane-agent/processor/lookupprocessor" + "github.com/observiq/bindplane-agent/processor/marshalprocessor" "github.com/observiq/bindplane-agent/processor/maskprocessor" "github.com/observiq/bindplane-agent/processor/metricextractprocessor" "github.com/observiq/bindplane-agent/processor/metricstatsprocessor" @@ -66,6 +67,7 @@ var defaultProcessors = []processor.Factory{ logdeduplicationprocessor.NewFactory(), logstransformprocessor.NewFactory(), lookupprocessor.NewFactory(), + marshalprocessor.NewFactory(), maskprocessor.NewFactory(), memorylimiterprocessor.NewFactory(), metricextractprocessor.NewFactory(), diff --git a/go.mod b/go.mod index 2bac203ee..a47b5cbb6 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/observiq/bindplane-agent/processor/logcountprocessor v1.60.0 github.com/observiq/bindplane-agent/processor/logdeduplicationprocessor v1.60.0 github.com/observiq/bindplane-agent/processor/lookupprocessor v1.60.0 + github.com/observiq/bindplane-agent/processor/marshalprocessor v1.60.0 github.com/observiq/bindplane-agent/processor/maskprocessor v1.60.0 github.com/observiq/bindplane-agent/processor/metricextractprocessor v1.60.0 github.com/observiq/bindplane-agent/processor/metricstatsprocessor v1.60.0 @@ -797,6 +798,8 @@ replace github.com/observiq/bindplane-agent/processor/throughputmeasurementproce replace github.com/observiq/bindplane-agent/processor/samplingprocessor => ./processor/samplingprocessor +replace github.com/observiq/bindplane-agent/processor/marshalprocessor => ./processor/marshalprocessor + replace github.com/observiq/bindplane-agent/processor/maskprocessor => ./processor/maskprocessor replace github.com/observiq/bindplane-agent/processor/logcountprocessor => ./processor/logcountprocessor diff --git a/processor/marshalprocessor/README.md b/processor/marshalprocessor/README.md new file mode 100644 index 000000000..d3172e1c5 --- /dev/null +++ b/processor/marshalprocessor/README.md @@ -0,0 +1,192 @@ +# Marshal Processor + +This processor is used to marshal parsed logs into JSON or KV format. + +This processor is intended to be wrapped into the Marshal processor in Bindplane. + +NOTE: XML support is in progress and not yet available. + +## Supported pipelines + +- Logs + +## How It Works + +1. This processor expects its input to contain a parsed log body. It will marshal the body fields only; if additional fields from the log are desired, they must first be moved to the body. + +2. The body can be marshaled to string-encoded JSON or KV. + + - For KV: + - Fields will be converted to "key1=value1 key2=value2 key3=value3..." if no separators are configured + - The parsed fields should be flattened first so that every key is at the top level + - If fields are not flattened, the nested fields will be converted to "nested=[k1=v1,k2=v2]..." if no map separators are configured + - If any key or value contains characters that conflict with the separators, they will be wrapped in `"` and any `"` inside them will be escaped + - Arrays will simply be stringified + +3. The output of this processor will be the same as the input, but with a modified log body. Any body incompatible with the marshal type will be unchanged. + +## Configuration + +| Field | Type | Default | Description | +| --------------------- | ------ | ------- | --------------------------------------------- | +| marshal_to | string | "" | The format to marshal into. Can be JSON or KV | +| kv_separator | rune | "=" | The separator between key and value | +| kv_pair_separator | rune | " " | The separator between KV pairs | +| map_kv_separator | rune | "=" | The separator between nested KV pairs | +| map_kv_pair_separator | rune | "," | The separator between nested KV pairs | + +## Example Config for JSON + +```yaml +receivers: + otlp: +processors: + transform: + marshal: + marshal_to: "JSON" +exporters: + chronicle: +service: + pipelines: + logs: + receivers: [otlp] + processors: [transform, marshal] + exporters: [chronicle] +``` + +## Example config for KV + +```yaml +receivers: + otlp: +processors: + transform: + marshal: + marshal_to: "KV" + kv_separator: "," + kv_pair_separator: ":" +exporters: + chronicle: +service: + pipelines: + logs: + receivers: [otlp] + processors: [transform, marshal] + exporters: [chronicle] +``` + +## Example Output + +The parsed body will be replaced by a marshaled body. All other fields are untouched. + +### 1. Nested parsed body to JSON + +In the example below, "bindplane-otel-attributes" represents attributes that have been moved to the body. + +#### Parsed body + +``` +"body": { + "kvlistValue": { + "values": [ + { "key": "severity", "value": { "doubleValue": 155 } }, + { + "key": "nested", + "value": { + "kvlistValue": { + "values": [ + { "key": "n2", "value": { "doubleValue": 2 } }, + { "key": "n1", "value": { "doubleValue": 1 } } + ] + } + } + }, + { "key": "name", "value": { "stringValue": "test" } }, + { + "key": "bindplane-otel-attributes", + "value": { + "kvlistValue": { + "values": [ + { + "key": "baba", + "value": { "stringValue": "you" } + }, + { + "key": "host", + "value": { "stringValue": "myhost" } + } + ] + } + } + } + ] + } +}, + +``` + +#### JSON output + +``` +"body": { + "stringValue": { + { + "bindplane-otel-attributes": + { + "baba":"you", + "host":"myhost" + }, + "name":"test", + "nested": + { + "n1":1, + "n2":2 + }, + "severity":155 + } + } +} +``` + +### 2: Flattened parsed body to KV with default separators + +In the example below, flattening has already been done on the "nested" field and the "bindplane-otel-attributes" field. + +#### Parsed flattened body + +``` +"body": { + "kvlistValue": { + "values": [ + { "key": "severity", "value": { "doubleValue": 155 } }, + { + "key": "nested-n1", + "value": { "doubleValue": 1 } + }, + { + "key": "nested-n2", + "value": { "doubleValue": 2 } + }, + { "key": "name", "value": { "stringValue": "test" } }, + { + "key": "bindplane-otel-attributes-baba", + "value": { "stringValue": "you" } + }, + { + "key": "bindplane-otel-attributes-host", + "value": { "stringValue": "myhost" } + } + ] + } +} +``` + +#### KV output + +``` +"body": { + "stringValue": { + bindplane-otel-attributes-baba=you bindplane-otel-attributes-host=myhost name=test nested-n1=1 nested-n2=2 severity=155 + } +} +``` diff --git a/processor/marshalprocessor/config.go b/processor/marshalprocessor/config.go new file mode 100644 index 000000000..bae90b9ba --- /dev/null +++ b/processor/marshalprocessor/config.go @@ -0,0 +1,82 @@ +// Copyright observIQ, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package marshalprocessor provides a processor that marshals logs to a specified format. +package marshalprocessor + +import ( + "errors" + "strings" + + "go.opentelemetry.io/collector/component" + "go.uber.org/multierr" +) + +var errInvalidMarshalTo = errors.New("marshal_to must be JSON or KV") +var errXMLNotSupported = errors.New("XML not yet supported") +var errKVSeparatorsEqual = errors.New("kv_separator and kv_pair_separator must be different") +var errMapKVSeparatorsEqual = errors.New("map_kv_separator and map_kv_pair_separator must be different") + +const ( + defaultMarshalTo = "JSON" + defaultKVSeparator = '=' + defaultKVPairSeparator = ' ' + defaultMapKVSeparator = '=' + defaultMapKVPairSeparator = ',' +) + +// Config is the configuration for the processor +type Config struct { + MarshalTo string `mapstructure:"marshal_to"` // MarshalTo is either JSON or KV + KVSeparator rune `mapstructure:"kv_separator"` + KVPairSeparator rune `mapstructure:"kv_pair_separator"` + MapKVSeparator rune `mapstructure:"map_kv_separator"` + MapKVPairSeparator rune `mapstructure:"map_kv_pair_separator"` +} + +// Validate validates the processor configuration +func (cfg Config) Validate() error { + var errs error + + // Validate MarshalTo choice + switch strings.ToUpper(cfg.MarshalTo) { + case "JSON": + case "XML": + errs = multierr.Append(errs, errXMLNotSupported) + case "KV": + // Validate KV separators, which must be different from each other + if cfg.KVSeparator == cfg.KVPairSeparator && cfg.KVSeparator != 0 { + errs = multierr.Append(errs, errKVSeparatorsEqual) + } + // Validate Map KV separators, which must be different from each other (but can match KV separators) + if cfg.MapKVSeparator == cfg.MapKVPairSeparator && cfg.MapKVSeparator != 0 { + errs = multierr.Append(errs, errMapKVSeparatorsEqual) + } + default: + errs = multierr.Append(errs, errInvalidMarshalTo) + } + + return errs +} + +// createDefaultConfig returns the default config for the processor. +func createDefaultConfig() component.Config { + return &Config{ + MarshalTo: defaultMarshalTo, + KVSeparator: defaultKVSeparator, + KVPairSeparator: defaultKVPairSeparator, + MapKVSeparator: defaultMapKVSeparator, + MapKVPairSeparator: defaultMapKVPairSeparator, + } +} diff --git a/processor/marshalprocessor/config_test.go b/processor/marshalprocessor/config_test.go new file mode 100644 index 000000000..79d21ea76 --- /dev/null +++ b/processor/marshalprocessor/config_test.go @@ -0,0 +1,177 @@ +// Copyright observIQ, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package marshalprocessor + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCreateDefaultProcessorConfig(t *testing.T) { + cfg := createDefaultConfig().(*Config) + require.Equal(t, defaultMarshalTo, cfg.MarshalTo) + require.Equal(t, defaultKVSeparator, cfg.KVSeparator) + require.Equal(t, defaultKVPairSeparator, cfg.KVPairSeparator) + require.Equal(t, defaultMapKVSeparator, cfg.MapKVSeparator) + require.Equal(t, defaultMapKVPairSeparator, cfg.MapKVPairSeparator) +} + +func TestConfigValidate(t *testing.T) { + testCases := []struct { + desc string + cfg *Config + expectedErr error + }{ + { + desc: "default", + cfg: createDefaultConfig().(*Config), + expectedErr: nil, + }, + { + desc: "JSON", + cfg: &Config{ + MarshalTo: "JSON", + }, + expectedErr: nil, + }, + { + desc: "XML", + cfg: &Config{ + MarshalTo: "XML", + }, + expectedErr: errXMLNotSupported, + }, + { + desc: "KV", + cfg: &Config{ + MarshalTo: "KV", + }, + expectedErr: nil, + }, + { + desc: "JSON lowercase", + cfg: &Config{ + MarshalTo: "json", + }, + expectedErr: nil, + }, + { + desc: "XML lowercase", + cfg: &Config{ + MarshalTo: "xml", + }, + expectedErr: errXMLNotSupported, + }, + { + desc: "KV lowercase", + cfg: &Config{ + MarshalTo: "kv", + }, + expectedErr: nil, + }, + { + desc: "error", + cfg: &Config{ + MarshalTo: "TOML", + }, + expectedErr: errInvalidMarshalTo, + }, + { + desc: "KV separator fields do not cause a validation error if not marshaling to KV", + cfg: &Config{ + MarshalTo: "JSON", + KVSeparator: ':', + KVPairSeparator: ':', + MapKVSeparator: '!', + MapKVPairSeparator: '!', + }, + expectedErr: nil, + }, + { + desc: "Identical KV separator fields are not allowed", + cfg: &Config{ + MarshalTo: "KV", + KVSeparator: ':', + KVPairSeparator: ':', + }, + expectedErr: errKVSeparatorsEqual, + }, + { + desc: "Identical KV separator fields are not allowed with default KVPairSeparator", + cfg: &Config{ + MarshalTo: "KV", + KVSeparator: ' ', + KVPairSeparator: ' ', + }, + expectedErr: errKVSeparatorsEqual, + }, + { + desc: "Identical KV separator fields are not allowed with default KVSeparator", + cfg: &Config{ + MarshalTo: "KV", + KVSeparator: '=', + KVPairSeparator: '=', + }, + expectedErr: errKVSeparatorsEqual, + }, + { + desc: "Identical Map KV separator fields are not allowed", + cfg: &Config{ + MarshalTo: "KV", + MapKVSeparator: ':', + MapKVPairSeparator: ':', + }, + expectedErr: errMapKVSeparatorsEqual, + }, + { + desc: "Identical Map KV separator fields are not allowed with default MapKVPairSeparator", + cfg: &Config{ + MarshalTo: "KV", + MapKVSeparator: ',', + MapKVPairSeparator: ',', + }, + expectedErr: errMapKVSeparatorsEqual, + }, + { + desc: "Identical Map KV separator fields are not allowed with default MapKVSeparator", + cfg: &Config{ + MarshalTo: "KV", + MapKVSeparator: '=', + MapKVPairSeparator: '=', + }, + expectedErr: errMapKVSeparatorsEqual, + }, + { + desc: "Map KV separators can match their KV counterparts", + cfg: &Config{ + MarshalTo: "KV", + KVSeparator: ':', + KVPairSeparator: ' ', + MapKVSeparator: ':', + MapKVPairSeparator: ' ', + }, + expectedErr: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + actualErr := tc.cfg.Validate() + assert.Equal(t, tc.expectedErr, actualErr) + }) + } +} diff --git a/processor/marshalprocessor/doc.go b/processor/marshalprocessor/doc.go new file mode 100644 index 000000000..29cd17b5e --- /dev/null +++ b/processor/marshalprocessor/doc.go @@ -0,0 +1,18 @@ +// Copyright observIQ, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:generate mdatagen metadata.yaml + +// Package marshalprocessor provides a processor that marshals logs to a specified format. +package marshalprocessor // import "github.com/observiq/bindplane-agent/processor/marshalprocessor" diff --git a/processor/marshalprocessor/factory.go b/processor/marshalprocessor/factory.go new file mode 100644 index 000000000..f637da080 --- /dev/null +++ b/processor/marshalprocessor/factory.go @@ -0,0 +1,50 @@ +// Copyright observIQ, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package marshalprocessor + +import ( + "context" + + "github.com/observiq/bindplane-agent/processor/marshalprocessor/internal/metadata" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/processor" + "go.opentelemetry.io/collector/processor/processorhelper" +) + +var ( + consumerCapabilities = consumer.Capabilities{MutatesData: true} +) + +// NewFactory creates a new ProcessorFactory with default configuration +func NewFactory() processor.Factory { + return processor.NewFactory( + metadata.Type, + createDefaultConfig, + processor.WithLogs(createLogsProcessor, metadata.LogsStability), + ) +} + +func createLogsProcessor( + ctx context.Context, + set processor.Settings, + cfg component.Config, + nextConsumer consumer.Logs, +) (processor.Logs, error) { + oCfg := cfg.(*Config) + tmp := newMarshalProcessor(set.Logger, oCfg) + + return processorhelper.NewLogsProcessor(ctx, set, cfg, nextConsumer, tmp.processLogs, processorhelper.WithCapabilities(consumerCapabilities)) +} diff --git a/processor/marshalprocessor/factory_test.go b/processor/marshalprocessor/factory_test.go new file mode 100644 index 000000000..2787d3d4b --- /dev/null +++ b/processor/marshalprocessor/factory_test.go @@ -0,0 +1,39 @@ +// Copyright observIQ, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package marshalprocessor + +import ( + "testing" + + "github.com/observiq/bindplane-agent/processor/marshalprocessor/internal/metadata" + "github.com/stretchr/testify/require" +) + +func TestNewFactory(t *testing.T) { + factory := NewFactory() + require.Equal(t, metadata.Type, factory.Type()) + + expectedCfg := &Config{ + MarshalTo: "JSON", + KVSeparator: '=', + KVPairSeparator: ' ', + MapKVSeparator: '=', + MapKVPairSeparator: ',', + } + + cfg, ok := factory.CreateDefaultConfig().(*Config) + require.True(t, ok) + require.Equal(t, expectedCfg, cfg) +} diff --git a/processor/marshalprocessor/generated_component_test.go b/processor/marshalprocessor/generated_component_test.go new file mode 100644 index 000000000..346262f20 --- /dev/null +++ b/processor/marshalprocessor/generated_component_test.go @@ -0,0 +1,135 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package marshalprocessor + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/confmap/confmaptest" + "go.opentelemetry.io/collector/consumer/consumertest" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.opentelemetry.io/collector/pdata/ptrace" + "go.opentelemetry.io/collector/processor" + "go.opentelemetry.io/collector/processor/processortest" +) + +func TestComponentFactoryType(t *testing.T) { + require.Equal(t, "marshal", NewFactory().Type().String()) +} + +func TestComponentConfigStruct(t *testing.T) { + require.NoError(t, componenttest.CheckConfigStruct(NewFactory().CreateDefaultConfig())) +} + +func TestComponentLifecycle(t *testing.T) { + factory := NewFactory() + + tests := []struct { + name string + createFn func(ctx context.Context, set processor.Settings, cfg component.Config) (component.Component, error) + }{ + + { + name: "logs", + createFn: func(ctx context.Context, set processor.Settings, cfg component.Config) (component.Component, error) { + return factory.CreateLogsProcessor(ctx, set, cfg, consumertest.NewNop()) + }, + }, + } + + cm, err := confmaptest.LoadConf("metadata.yaml") + require.NoError(t, err) + cfg := factory.CreateDefaultConfig() + sub, err := cm.Sub("tests::config") + require.NoError(t, err) + require.NoError(t, sub.Unmarshal(&cfg)) + + for _, test := range tests { + t.Run(test.name+"-shutdown", func(t *testing.T) { + c, err := test.createFn(context.Background(), processortest.NewNopSettings(), cfg) + require.NoError(t, err) + err = c.Shutdown(context.Background()) + require.NoError(t, err) + }) + t.Run(test.name+"-lifecycle", func(t *testing.T) { + c, err := test.createFn(context.Background(), processortest.NewNopSettings(), cfg) + require.NoError(t, err) + host := componenttest.NewNopHost() + err = c.Start(context.Background(), host) + require.NoError(t, err) + require.NotPanics(t, func() { + switch test.name { + case "logs": + e, ok := c.(processor.Logs) + require.True(t, ok) + logs := generateLifecycleTestLogs() + if !e.Capabilities().MutatesData { + logs.MarkReadOnly() + } + err = e.ConsumeLogs(context.Background(), logs) + case "metrics": + e, ok := c.(processor.Metrics) + require.True(t, ok) + metrics := generateLifecycleTestMetrics() + if !e.Capabilities().MutatesData { + metrics.MarkReadOnly() + } + err = e.ConsumeMetrics(context.Background(), metrics) + case "traces": + e, ok := c.(processor.Traces) + require.True(t, ok) + traces := generateLifecycleTestTraces() + if !e.Capabilities().MutatesData { + traces.MarkReadOnly() + } + err = e.ConsumeTraces(context.Background(), traces) + } + }) + require.NoError(t, err) + err = c.Shutdown(context.Background()) + require.NoError(t, err) + }) + } +} + +func generateLifecycleTestLogs() plog.Logs { + logs := plog.NewLogs() + rl := logs.ResourceLogs().AppendEmpty() + rl.Resource().Attributes().PutStr("resource", "R1") + l := rl.ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + l.Body().SetStr("test log message") + l.SetTimestamp(pcommon.NewTimestampFromTime(time.Now())) + return logs +} + +func generateLifecycleTestMetrics() pmetric.Metrics { + metrics := pmetric.NewMetrics() + rm := metrics.ResourceMetrics().AppendEmpty() + rm.Resource().Attributes().PutStr("resource", "R1") + m := rm.ScopeMetrics().AppendEmpty().Metrics().AppendEmpty() + m.SetName("test_metric") + dp := m.SetEmptyGauge().DataPoints().AppendEmpty() + dp.Attributes().PutStr("test_attr", "value_1") + dp.SetIntValue(123) + dp.SetTimestamp(pcommon.NewTimestampFromTime(time.Now())) + return metrics +} + +func generateLifecycleTestTraces() ptrace.Traces { + traces := ptrace.NewTraces() + rs := traces.ResourceSpans().AppendEmpty() + rs.Resource().Attributes().PutStr("resource", "R1") + span := rs.ScopeSpans().AppendEmpty().Spans().AppendEmpty() + span.Attributes().PutStr("test_attr", "value_1") + span.SetName("test_span") + span.SetStartTimestamp(pcommon.NewTimestampFromTime(time.Now().Add(-1 * time.Second))) + span.SetEndTimestamp(pcommon.NewTimestampFromTime(time.Now())) + return traces +} diff --git a/processor/marshalprocessor/generated_package_test.go b/processor/marshalprocessor/generated_package_test.go new file mode 100644 index 000000000..54459b593 --- /dev/null +++ b/processor/marshalprocessor/generated_package_test.go @@ -0,0 +1,13 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package marshalprocessor + +import ( + "testing" + + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} diff --git a/processor/marshalprocessor/go.mod b/processor/marshalprocessor/go.mod new file mode 100644 index 000000000..be9c8d25e --- /dev/null +++ b/processor/marshalprocessor/go.mod @@ -0,0 +1,67 @@ +module github.com/observiq/bindplane-agent/processor/marshalprocessor + +go 1.22.0 + +toolchain go1.22.6 + +require ( + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.108.0 + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.108.0 + github.com/stretchr/testify v1.9.0 + go.opentelemetry.io/collector/component v0.106.1 + go.opentelemetry.io/collector/confmap v0.106.1 + go.opentelemetry.io/collector/consumer v0.106.1 + go.opentelemetry.io/collector/consumer/consumertest v0.106.1 + go.opentelemetry.io/collector/pdata v1.14.1 + go.opentelemetry.io/collector/processor v0.106.1 + go.uber.org/goleak v1.3.0 + go.uber.org/multierr v1.11.0 + go.uber.org/zap v1.27.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-viper/mapstructure/v2 v2.0.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/knadh/koanf/maps v0.1.1 // indirect + github.com/knadh/koanf/providers/confmap v0.1.0 // indirect + github.com/knadh/koanf/v2 v2.1.1 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.108.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.19.1 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + go.opentelemetry.io/collector v0.106.1 // indirect + go.opentelemetry.io/collector/config/configtelemetry v0.106.1 // indirect + go.opentelemetry.io/collector/consumer/consumerprofiles v0.106.1 // indirect + go.opentelemetry.io/collector/featuregate v1.12.0 // indirect + go.opentelemetry.io/collector/internal/globalgates v0.106.1 // indirect + go.opentelemetry.io/collector/pdata/pprofile v0.106.1 // indirect + go.opentelemetry.io/collector/pdata/testdata v0.106.1 // indirect + go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/prometheus v0.50.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/sdk v1.28.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.28.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/grpc v1.65.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/processor/marshalprocessor/go.sum b/processor/marshalprocessor/go.sum new file mode 100644 index 000000000..1d6b12dee --- /dev/null +++ b/processor/marshalprocessor/go.sum @@ -0,0 +1,158 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-viper/mapstructure/v2 v2.0.0 h1:dhn8MZ1gZ0mzeodTG3jt5Vj/o87xZKuNAprG2mQfMfc= +github.com/go-viper/mapstructure/v2 v2.0.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= +github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU= +github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU= +github.com/knadh/koanf/v2 v2.1.1 h1:/R8eXqasSTsmDCsAyYj+81Wteg8AqrV9CP6gvsTsOmM= +github.com/knadh/koanf/v2 v2.1.1/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.108.0 h1:WbZ/Oq3z5x/PQAwzB0vMhP4QNc4nI1G22H3AMAAbMjs= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.108.0/go.mod h1:3ku/cfl0FXMSc/dc9DGrhABhE6/AoYArKtl3I9QEp28= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.108.0 h1:so+V3rT18oyHJmPs5lBNBLlU8tnHU9h/tkA3Q7q8m7c= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.108.0/go.mod h1:G+N43ID1sP2CnffxkYdMyuJpep2UcGQUyq4HiAmcYSw= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.108.0 h1:vU6a7EKHBu80err/1SCn+8fLpSCdR1PoSzdyydXDcjQ= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.108.0/go.mod h1:dHSW1ec9ZHZ/92+NoLEJd5UaL4tRVmnn+DvEQqdJ7f0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/collector v0.106.1 h1:ZSQMpFGzFP3RILe1/+K80kCCT2ahn3MKt5e3u0Yz7Rs= +go.opentelemetry.io/collector v0.106.1/go.mod h1:1FabMxWLluLNcC0dq8cI01GaE6t6fYxE6Oxuf8u7AGQ= +go.opentelemetry.io/collector/component v0.106.1 h1:6Xp4tKqnd/JkJDG/C4p1hto+Y5zvk5FwqZIdMCPzZlA= +go.opentelemetry.io/collector/component v0.106.1/go.mod h1:KiVE/5ZayuLlDJTe7mHqHRCn/5LrmF99C7/mKe54mWA= +go.opentelemetry.io/collector/config/configtelemetry v0.106.1 h1:A8nwYnI6brfur5KPFC8GtVX/49pByvVoKSgO4qPXBqg= +go.opentelemetry.io/collector/config/configtelemetry v0.106.1/go.mod h1:WxWKNVAQJg/Io1nA3xLgn/DWLE/W1QOB2+/Js3ACi40= +go.opentelemetry.io/collector/confmap v0.106.1 h1:R7HQIPDRPOEwauBeJUlkT8Elc5f0KQr/s/kQfZi05t0= +go.opentelemetry.io/collector/confmap v0.106.1/go.mod h1:iWdWgvxRYSHdAt5ySgPJq/i6fQMKGNnP5Pt7jOfTXno= +go.opentelemetry.io/collector/consumer v0.106.1 h1:+AQ/Kmoc/g0WP8thwymNkXk1jeWsHDK6XyYfdezcxcc= +go.opentelemetry.io/collector/consumer v0.106.1/go.mod h1:oy6pR/v5o/N9cxsICskyt//bU8k8EG0JeOO1MTDfs5A= +go.opentelemetry.io/collector/consumer/consumerprofiles v0.106.1 h1:uxQjWm2XE7d1OncQDM9tL1ha+otGt1HjoRYIcQRMOfQ= +go.opentelemetry.io/collector/consumer/consumerprofiles v0.106.1/go.mod h1:xQScBf9/PORFaYM6JVPOr7/TcRVEuKcW5XbAXfJByRs= +go.opentelemetry.io/collector/consumer/consumertest v0.106.1 h1:hDdFeVjCLIJ6iLfbiYcV9s+4iboFXbkJ/k3h09qusPw= +go.opentelemetry.io/collector/consumer/consumertest v0.106.1/go.mod h1:WRTYnQ8bYHQrEN6eJZ80oC4pNI7VeDRdsTZI6xs9o5M= +go.opentelemetry.io/collector/featuregate v1.12.0 h1:l5WbV2vMQd2bL8ubfGrbKNtZaeJRckE12CTHvRe47Tw= +go.opentelemetry.io/collector/featuregate v1.12.0/go.mod h1:PsOINaGgTiFc+Tzu2K/X2jP+Ngmlp7YKGV1XrnBkH7U= +go.opentelemetry.io/collector/internal/globalgates v0.106.1 h1:0NQHTcykmYNDsNKObJ2XocGCv3WUAQZppfP3o6hZUIA= +go.opentelemetry.io/collector/internal/globalgates v0.106.1/go.mod h1:Z5US6O2xkZAtxVSSBnHAPFZwPhFoxlyKLUvS67Vx4gc= +go.opentelemetry.io/collector/pdata v1.14.1 h1:wXZjtQA7Vy5HFqco+yA95ENyMQU5heBB1IxMHQf6mUk= +go.opentelemetry.io/collector/pdata v1.14.1/go.mod h1:z1dTjwwtcoXxZx2/nkHysjxMeaxe9pEmYTEr4SMNIx8= +go.opentelemetry.io/collector/pdata/pprofile v0.106.1 h1:nOLo25YnluNi+zAbU7G24RN86cJ1/EZJc6VEayBlOPo= +go.opentelemetry.io/collector/pdata/pprofile v0.106.1/go.mod h1:chr7lMJIzyXkccnPRkIPhyXtqLZLSReZYhwsggOGEfg= +go.opentelemetry.io/collector/pdata/testdata v0.106.1 h1:JUyLAwKD8o/9jgkBi16zOClxOyY028A7XIXHPV4mNmM= +go.opentelemetry.io/collector/pdata/testdata v0.106.1/go.mod h1:ghdz2RDEzsfigW0J+9oqA4fGmQJ/DJYUhE3vYU6JfhM= +go.opentelemetry.io/collector/processor v0.106.1 h1:W/SmNRkGLf6dOWjdqU5WlDnPLDQJRyHZxI6X8IQwFec= +go.opentelemetry.io/collector/processor v0.106.1/go.mod h1:D4Ni5zbK/QtkIxSbDEZanUcLN9zM3JnlU9hc3Qm/o6I= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/exporters/prometheus v0.50.0 h1:2Ewsda6hejmbhGFyUvWZjUThC98Cf8Zy6g0zkIimOng= +go.opentelemetry.io/otel/exporters/prometheus v0.50.0/go.mod h1:pMm5PkUo5YwbLiuEf7t2xg4wbP0/eSJrMxIMxKosynY= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= +go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= +go.opentelemetry.io/otel/sdk/metric v1.28.0 h1:OkuaKgKrgAbYrrY0t92c+cC+2F6hsFNnCQArXCKlg08= +go.opentelemetry.io/otel/sdk/metric v1.28.0/go.mod h1:cWPjykihLAPvXKi4iZc1dpER3Jdq2Z0YLse3moQUCpg= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/processor/marshalprocessor/internal/metadata/generated_status.go b/processor/marshalprocessor/internal/metadata/generated_status.go new file mode 100644 index 000000000..0fe4053d8 --- /dev/null +++ b/processor/marshalprocessor/internal/metadata/generated_status.go @@ -0,0 +1,15 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "go.opentelemetry.io/collector/component" +) + +var ( + Type = component.MustNewType("marshal") +) + +const ( + LogsStability = component.StabilityLevelDevelopment +) diff --git a/processor/marshalprocessor/metadata.yaml b/processor/marshalprocessor/metadata.yaml new file mode 100644 index 000000000..bacfab594 --- /dev/null +++ b/processor/marshalprocessor/metadata.yaml @@ -0,0 +1,11 @@ +type: marshal + +status: + class: processor + stability: + development: [logs] + distributions: [observiq] + codeowners: + active: [kuiperda] +tests: + config: diff --git a/processor/marshalprocessor/processor.go b/processor/marshalprocessor/processor.go new file mode 100644 index 000000000..fcdd0505a --- /dev/null +++ b/processor/marshalprocessor/processor.go @@ -0,0 +1,143 @@ +// Copyright observIQ, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package marshalprocessor + +import ( + "context" + "fmt" + "sort" + "strings" + + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + "go.uber.org/zap" +) + +type marshalProcessor struct { + logger *zap.Logger + marshalTo string + kvSeparator string + kvPairSeparator string + mapKVSeparator string + mapKVPairSeparator string +} + +func newMarshalProcessor(logger *zap.Logger, cfg *Config) *marshalProcessor { + return &marshalProcessor{ + logger: logger, + marshalTo: cfg.MarshalTo, + kvSeparator: string(cfg.KVSeparator), + kvPairSeparator: string(cfg.KVPairSeparator), + mapKVSeparator: string(cfg.MapKVSeparator), + mapKVPairSeparator: string(cfg.MapKVPairSeparator), + } +} + +func (mp *marshalProcessor) processLogs(_ context.Context, ld plog.Logs) (plog.Logs, error) { + for i := 0; i < ld.ResourceLogs().Len(); i++ { + resourceLog := ld.ResourceLogs().At(i) + for j := 0; j < resourceLog.ScopeLogs().Len(); j++ { + scopeLog := resourceLog.ScopeLogs().At(j) + for k := 0; k < scopeLog.LogRecords().Len(); k++ { + logRecord := scopeLog.LogRecords().At(k) + logBody := logRecord.Body() + // If body is not a map, skip that log + if logBody.Type() != pcommon.ValueTypeMap { + mp.logger.Warn("Non map body not supported", zap.Any("body", logBody)) + continue + } + switch strings.ToUpper(mp.marshalTo) { + case "JSON": + jsonBody := logBody.AsString() + logBody.SetStr(jsonBody) + case "XML": + return ld, fmt.Errorf("XML not yet supported") + case "KV": + kvBody := mp.convertMapToKV(logBody.Map(), false) + logBody.SetStr(kvBody) + default: + return ld, fmt.Errorf("Unrecognized format to marshal to: %s", mp.marshalTo) + } + } + } + } + + return ld, nil +} + +// convertMapToKV recursively converts a map to a key value string +func (mp *marshalProcessor) convertMapToKV(logBody pcommon.Map, inNestedValue bool) string { + var kvStrings []string + var keyValues []struct { + key string + val pcommon.Value + } + + // Sort by keys + logBody.Range(func(k string, v pcommon.Value) bool { + keyValues = append(keyValues, struct { + key string + val pcommon.Value + }{key: k, val: v}) + return true + }) + sort.Slice(keyValues, func(i, j int) bool { + return keyValues[i].key < keyValues[j].key + }) + + // Convert KV pairs + for _, kv := range keyValues { + k := mp.escapeAndQuoteKV(kv.key, inNestedValue) + + var vStr string + switch kv.val.Type() { + case pcommon.ValueTypeMap: + vStr = mp.convertMapToKV(kv.val.Map(), true) + vStr = `[` + vStr + `]` + if !inNestedValue && (mp.kvSeparator == mp.mapKVSeparator || mp.kvPairSeparator == mp.mapKVPairSeparator) { + vStr = `"` + vStr + `"` + } + default: + vStr = mp.escapeAndQuoteKV(kv.val.AsString(), inNestedValue) + } + + if !inNestedValue { + kvStrings = append(kvStrings, fmt.Sprintf("%s%s%v", k, mp.kvSeparator, vStr)) + } else { + kvStrings = append(kvStrings, fmt.Sprintf("%s%s%v", k, mp.mapKVSeparator, vStr)) + } + } + + if !inNestedValue { + return strings.Join(kvStrings, mp.kvPairSeparator) + } + return strings.Join(kvStrings, mp.mapKVPairSeparator) +} + +func (mp marshalProcessor) escapeAndQuoteKV(s string, inNestedValue bool) string { + if !inNestedValue { + s = strings.ReplaceAll(s, `"`, `\"`) + if strings.Contains(s, mp.kvPairSeparator) || strings.Contains(s, mp.kvSeparator) { + s = `"` + s + `"` + } + } else { + s = strings.ReplaceAll(s, `"`, `\\\"`) + if strings.ContainsAny(s, mp.kvPairSeparator+mp.kvSeparator+mp.mapKVPairSeparator+mp.mapKVSeparator+`[]`) { + s = `\"` + s + `\"` + } + } + + return s +} diff --git a/processor/marshalprocessor/processor_test.go b/processor/marshalprocessor/processor_test.go new file mode 100644 index 000000000..b46798301 --- /dev/null +++ b/processor/marshalprocessor/processor_test.go @@ -0,0 +1,187 @@ +// Copyright observIQ, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package marshalprocessor + +import ( + "context" + "path/filepath" + "testing" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden" + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest/plogtest" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +func Test_processLogs(t *testing.T) { + testCases := []struct { + desc string + marshalTo string + kvSeparator rune + kvPairSeparator rune + mapKVSeparator rune + mapKVPairSeparator rune + inputFilePath string + expectedOutputFilePath string + }{ + { + desc: "Valid - Parsed body to JSON", + marshalTo: "JSON", + inputFilePath: "parsed-log.json", + expectedOutputFilePath: `parsed-log-json.json`, + }, + { + desc: "Invalid - String body to JSON", + marshalTo: "JSON", + inputFilePath: "string-log.json", + expectedOutputFilePath: "string-log.json", + }, + { + desc: "Invalid - String body to KV", + marshalTo: "KV", + inputFilePath: "string-log.json", + expectedOutputFilePath: "string-log.json", + }, + { + desc: "Invalid - String body to XML", + marshalTo: "XML", + inputFilePath: "string-log.json", + expectedOutputFilePath: "string-log.json", + }, + { + desc: "Valid - Parsed and flattened body to KV with default separators", + marshalTo: "KV", + inputFilePath: "parsed-flattened-log.json", + expectedOutputFilePath: "parsed-flattened-log-kv.json", + }, + { + desc: "Valid - Parsed nested body to KV with default separators", // not recommended to use this unflattened format but technically valid + marshalTo: "KV", + inputFilePath: "parsed-log.json", + expectedOutputFilePath: "parsed-log-kv.json", + }, + { + desc: "Valid - Parsed deeply nested body to KV with default separators", // not recommended to use this unflattened format but technically valid + marshalTo: "KV", + inputFilePath: "parsed-log-deeply-nested.json", + expectedOutputFilePath: "parsed-log-deeply-nested-kv.json", + }, + { + desc: "Valid - Parsed deeply nested body to KV with custom Map separators", // not recommended to use this unflattened format but technically valid + marshalTo: "KV", + mapKVSeparator: ':', + mapKVPairSeparator: '|', + inputFilePath: "parsed-log-deeply-nested.json", + expectedOutputFilePath: "parsed-log-deeply-nested-kv-map.json", + }, + { + desc: "Valid - Parsed deeply nested body to KV with default separators and separators present in nested map", // not recommended to use this unflattened format but technically valid + marshalTo: "KV", + inputFilePath: "parsed-log-deeply-nested-with-separators.json", + expectedOutputFilePath: "parsed-log-deeply-nested-with-separators-kv.json", + }, + { + desc: "Valid - Parsed deeply nested body to KV with custom Map separators and map separators present in nested map", // not recommended to use this unflattened format but technically valid + marshalTo: "KV", + mapKVSeparator: ':', + mapKVPairSeparator: '|', + inputFilePath: "parsed-log-deeply-nested-with-separators-map.json", + expectedOutputFilePath: "parsed-log-deeply-nested-with-separators-kv-map.json", + }, + { + desc: "Valid - Parsed deeply nested body to KV with normal KV separators in nested map vals but not the custom map separators in them", // not recommended to use this unflattened format but technically valid + marshalTo: "KV", + mapKVSeparator: ':', + mapKVPairSeparator: '|', + inputFilePath: "parsed-log-deeply-nested-with-separators-map-2.json", + expectedOutputFilePath: "parsed-log-deeply-nested-with-separators-kv-map-2.json", + }, + { + desc: "Valid - Parsed and flattened body to KV with custom pair separator", + marshalTo: "KV", + kvPairSeparator: '|', + inputFilePath: "parsed-flattened-log.json", + expectedOutputFilePath: "parsed-flattened-log-kv-pipe.json", + }, + { + desc: "Valid - Parsed and flattened body to KV with custom separator", + marshalTo: "KV", + kvSeparator: '+', + inputFilePath: "parsed-flattened-log.json", + expectedOutputFilePath: "parsed-flattened-log-kv-plus.json", + }, + { + desc: "Valid - Parsed and flattened body to KV with default separators as part of the KV values", + marshalTo: "KV", + inputFilePath: "parsed-flattened-log-with-separators.json", + expectedOutputFilePath: "parsed-flattened-log-kv-separators.json", + }, + { + desc: "Valid - Parsed and flattened body to KV with custom separators as part of the KV values", + marshalTo: "KV", + kvPairSeparator: ',', + kvSeparator: ':', + inputFilePath: "parsed-flattened-log-with-custom-separators.json", + expectedOutputFilePath: "parsed-flattened-log-kv-custom-separators.json", + }, + { + desc: "Valid - Parsed and flattened body to KV with default separators as part of the KV values and quotes inside the KV values", + marshalTo: "KV", + inputFilePath: "parsed-flattened-log-with-separators-and-quotes.json", + expectedOutputFilePath: "parsed-flattened-log-kv-separators-and-quotes.json", + }, + { + desc: "Valid - Parsed body with nested arrays and maps to KV with default separators", // not recommended to use this unflattened format but technically valid + marshalTo: "KV", + inputFilePath: "parsed-nested-log-with-slice.json", + expectedOutputFilePath: "parsed-nested-log-with-slice-kv.json", + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + cfg := &Config{ + MarshalTo: tc.marshalTo, + KVSeparator: tc.kvSeparator, + KVPairSeparator: tc.kvPairSeparator, + MapKVSeparator: tc.mapKVSeparator, + MapKVPairSeparator: tc.mapKVPairSeparator, + } + + if cfg.KVSeparator == 0 { + cfg.KVSeparator = defaultKVSeparator + } + if cfg.KVPairSeparator == 0 { + cfg.KVPairSeparator = defaultKVPairSeparator + } + if cfg.MapKVSeparator == 0 { + cfg.MapKVSeparator = defaultMapKVSeparator + } + if cfg.MapKVPairSeparator == 0 { + cfg.MapKVPairSeparator = defaultMapKVPairSeparator + } + + processor := newMarshalProcessor(zap.NewNop(), cfg) + inputlogs, err := golden.ReadLogs(filepath.Join("testdata", "input", tc.inputFilePath)) + require.NoError(t, err) + actual, err := processor.processLogs(context.Background(), inputlogs) + require.NoError(t, err) + expectedOutput, err := golden.ReadLogs(filepath.Join("testdata", "output", tc.expectedOutputFilePath)) + require.NoError(t, err) + + require.NoError(t, plogtest.CompareLogs(expectedOutput, actual)) + }) + } +} diff --git a/processor/marshalprocessor/testdata/input/parsed-flattened-log-with-custom-separators.json b/processor/marshalprocessor/testdata/input/parsed-flattened-log-with-custom-separators.json new file mode 100644 index 000000000..ab1e0b682 --- /dev/null +++ b/processor/marshalprocessor/testdata/input/parsed-flattened-log-with-custom-separators.json @@ -0,0 +1,53 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "kvlistValue": { + "values": [ + { "key": "severity", "value": { "doubleValue": 155 } }, + { + "key": "nested-n1:", + "value": { "doubleValue": 1 } + }, + { + "key": "nested-n2", + "value": { "doubleValue": 2 } + }, + { "key": "nam,e", "value": { "stringValue": "test:" } }, + { + "key": "bindplane-otel-attributes-baba", + "value": { "stringValue": "you" } + }, + { + "key": "bindplane-otel-attributes-host", + "value": { "stringValue": "myh,ost" } + } + ] + } + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/input/parsed-flattened-log-with-separators-and-quotes.json b/processor/marshalprocessor/testdata/input/parsed-flattened-log-with-separators-and-quotes.json new file mode 100644 index 000000000..70f4f462f --- /dev/null +++ b/processor/marshalprocessor/testdata/input/parsed-flattened-log-with-separators-and-quotes.json @@ -0,0 +1,53 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "kvlistValue": { + "values": [ + { "key": "s\"everity", "value": { "doubleValue": 155 } }, + { + "key": "nested-n1=", + "value": { "doubleValue": 1 } + }, + { + "key": "nested-n2", + "value": { "doubleValue": 2 } + }, + { "key": "nam e", "value": { "stringValue": "test=" } }, + { + "key": "bindplane-otel-attributes-baba", + "value": { "stringValue": "you" } + }, + { + "key": "bindplane-otel-attributes-host", + "value": { "stringValue": "myh \"ost" } + } + ] + } + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/input/parsed-flattened-log-with-separators.json b/processor/marshalprocessor/testdata/input/parsed-flattened-log-with-separators.json new file mode 100644 index 000000000..162db367f --- /dev/null +++ b/processor/marshalprocessor/testdata/input/parsed-flattened-log-with-separators.json @@ -0,0 +1,53 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "kvlistValue": { + "values": [ + { "key": "severity", "value": { "doubleValue": 155 } }, + { + "key": "nested-n1=", + "value": { "doubleValue": 1 } + }, + { + "key": "nested-n2", + "value": { "doubleValue": 2 } + }, + { "key": "nam e", "value": { "stringValue": "test=" } }, + { + "key": "bindplane-otel-attributes-baba", + "value": { "stringValue": "you" } + }, + { + "key": "bindplane-otel-attributes-host", + "value": { "stringValue": "myh ost" } + } + ] + } + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/input/parsed-flattened-log.json b/processor/marshalprocessor/testdata/input/parsed-flattened-log.json new file mode 100644 index 000000000..6a1644229 --- /dev/null +++ b/processor/marshalprocessor/testdata/input/parsed-flattened-log.json @@ -0,0 +1,53 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "kvlistValue": { + "values": [ + { "key": "severity", "value": { "doubleValue": 155 } }, + { + "key": "nested-n1", + "value": { "doubleValue": 1 } + }, + { + "key": "nested-n2", + "value": { "doubleValue": 2 } + }, + { "key": "name", "value": { "stringValue": "test" } }, + { + "key": "bindplane-otel-attributes-baba", + "value": { "stringValue": "you" } + }, + { + "key": "bindplane-otel-attributes-host", + "value": { "stringValue": "myhost" } + } + ] + } + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/input/parsed-log-deeply-nested-with-separators-map-2.json b/processor/marshalprocessor/testdata/input/parsed-log-deeply-nested-with-separators-map-2.json new file mode 100644 index 000000000..89bc9c2fc --- /dev/null +++ b/processor/marshalprocessor/testdata/input/parsed-log-deeply-nested-with-separators-map-2.json @@ -0,0 +1,86 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "kvlistValue": { + "values": [ + { "key": "severity", "value": { "doubleValue": 155 } }, + { + "key": "nested", + "value": { + "kvlistValue": { + "values": [ + { + "key": "n2", + "value": { + "kvlistValue": { + "values": [ + { + "key": "n3", + "value": { + "stringValue": "a not = great string" + } + }, + { + "key": "n4", + "value": { "doubleValue": 4 } + } + ] + } + } + }, + { + "key": "a not = great string", + "value": { "doubleValue": 1 } + } + ] + } + } + }, + { "key": "name", "value": { "stringValue": "test" } }, + { + "key": "bindplane-otel-attributes", + "value": { + "kvlistValue": { + "values": [ + { + "key": "baba", + "value": { "stringValue": "you" } + }, + { + "key": "host", + "value": { "stringValue": "myhost" } + } + ] + } + } + } + ] + } + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/input/parsed-log-deeply-nested-with-separators-map.json b/processor/marshalprocessor/testdata/input/parsed-log-deeply-nested-with-separators-map.json new file mode 100644 index 000000000..0f8c85946 --- /dev/null +++ b/processor/marshalprocessor/testdata/input/parsed-log-deeply-nested-with-separators-map.json @@ -0,0 +1,86 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "kvlistValue": { + "values": [ + { "key": "severity", "value": { "doubleValue": 155 } }, + { + "key": "nested", + "value": { + "kvlistValue": { + "values": [ + { + "key": "n2", + "value": { + "kvlistValue": { + "values": [ + { + "key": "n3", + "value": { + "stringValue": "a not :|| []]great string \"" + } + }, + { + "key": "n4", + "value": { "doubleValue": 4 } + } + ] + } + } + }, + { + "key": "a not :|| []]great string \"", + "value": { "doubleValue": 1 } + } + ] + } + } + }, + { "key": "name", "value": { "stringValue": "test" } }, + { + "key": "bindplane-otel-attributes", + "value": { + "kvlistValue": { + "values": [ + { + "key": "baba", + "value": { "stringValue": "you" } + }, + { + "key": "host", + "value": { "stringValue": "myhost" } + } + ] + } + } + } + ] + } + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/input/parsed-log-deeply-nested-with-separators.json b/processor/marshalprocessor/testdata/input/parsed-log-deeply-nested-with-separators.json new file mode 100644 index 000000000..b0a2c8055 --- /dev/null +++ b/processor/marshalprocessor/testdata/input/parsed-log-deeply-nested-with-separators.json @@ -0,0 +1,86 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "kvlistValue": { + "values": [ + { "key": "severity", "value": { "doubleValue": 155 } }, + { + "key": "nested", + "value": { + "kvlistValue": { + "values": [ + { + "key": "n2", + "value": { + "kvlistValue": { + "values": [ + { + "key": "n3", + "value": { + "stringValue": "a not =,, []]great string \"" + } + }, + { + "key": "n4", + "value": { "doubleValue": 4 } + } + ] + } + } + }, + { + "key": "a not =,, []]great string \"", + "value": { "doubleValue": 1 } + } + ] + } + } + }, + { "key": "name", "value": { "stringValue": "test" } }, + { + "key": "bindplane-otel-attributes", + "value": { + "kvlistValue": { + "values": [ + { + "key": "baba", + "value": { "stringValue": "you" } + }, + { + "key": "host", + "value": { "stringValue": "myhost" } + } + ] + } + } + } + ] + } + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/input/parsed-log-deeply-nested.json b/processor/marshalprocessor/testdata/input/parsed-log-deeply-nested.json new file mode 100644 index 000000000..09cb65ca9 --- /dev/null +++ b/processor/marshalprocessor/testdata/input/parsed-log-deeply-nested.json @@ -0,0 +1,81 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "kvlistValue": { + "values": [ + { "key": "severity", "value": { "doubleValue": 155 } }, + { + "key": "nested", + "value": { + "kvlistValue": { + "values": [ + { + "key": "n2", + "value": { + "kvlistValue": { + "values": [ + { + "key": "n3", + "value": { "doubleValue": 3 } + }, + { + "key": "n4", + "value": { "doubleValue": 4 } + } + ] + } + } + }, + { "key": "n1", "value": { "doubleValue": 1 } } + ] + } + } + }, + { "key": "name", "value": { "stringValue": "test" } }, + { + "key": "bindplane-otel-attributes", + "value": { + "kvlistValue": { + "values": [ + { + "key": "baba", + "value": { "stringValue": "you" } + }, + { + "key": "host", + "value": { "stringValue": "myhost" } + } + ] + } + } + } + ] + } + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/input/parsed-log.json b/processor/marshalprocessor/testdata/input/parsed-log.json new file mode 100644 index 000000000..8d7c57278 --- /dev/null +++ b/processor/marshalprocessor/testdata/input/parsed-log.json @@ -0,0 +1,65 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "kvlistValue": { + "values": [ + { "key": "severity", "value": { "doubleValue": 155 } }, + { + "key": "nested", + "value": { + "kvlistValue": { + "values": [ + { "key": "n2", "value": { "doubleValue": 2 } }, + { "key": "n1", "value": { "doubleValue": 1 } } + ] + } + } + }, + { "key": "name", "value": { "stringValue": "test" } }, + { + "key": "bindplane-otel-attributes", + "value": { + "kvlistValue": { + "values": [ + { + "key": "baba", + "value": { "stringValue": "you" } + }, + { + "key": "host", + "value": { "stringValue": "myhost" } + } + ] + } + } + } + ] + } + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/input/parsed-nested-log-with-slice.json b/processor/marshalprocessor/testdata/input/parsed-nested-log-with-slice.json new file mode 100644 index 000000000..d96f0c835 --- /dev/null +++ b/processor/marshalprocessor/testdata/input/parsed-nested-log-with-slice.json @@ -0,0 +1,97 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "kvlistValue": { + "values": [ + { "key": "severity", "value": { "doubleValue": 155 } }, + { + "key": "nested", + "value": { + "kvlistValue": { + "values": [ + { "key": "n2", "value": { "doubleValue": 2 } }, + { + "key": "n1", + "value": { + "arrayValue": { + "values": [ + { "doubleValue": 1 }, + { "doubleValue": 2 }, + { + "arrayValue": { + "values": [ + { "doubleValue": 1 }, + { "doubleValue": 2 } + ] + } + }, + { + "kvlistValue": { + "values": [ + { + "key": "n2", + "value": { "doubleValue": 2 } + }, + { + "key": "n1", + "value": { "doubleValue": 1 } + } + ] + } + } + ] + } + } + } + ] + } + } + }, + { "key": "name", "value": { "stringValue": "test" } }, + { + "key": "bindplane-otel-attributes", + "value": { + "kvlistValue": { + "values": [ + { + "key": "baba", + "value": { "stringValue": "you" } + }, + { + "key": "host", + "value": { "stringValue": "myhost" } + } + ] + } + } + } + ] + } + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/input/string-log.json b/processor/marshalprocessor/testdata/input/string-log.json new file mode 100644 index 000000000..796158045 --- /dev/null +++ b/processor/marshalprocessor/testdata/input/string-log.json @@ -0,0 +1,32 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "stringValue": "this log is a string that cannot be parsed" + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/output/parsed-flattened-log-kv-custom-separators.json b/processor/marshalprocessor/testdata/output/parsed-flattened-log-kv-custom-separators.json new file mode 100644 index 000000000..a89df7185 --- /dev/null +++ b/processor/marshalprocessor/testdata/output/parsed-flattened-log-kv-custom-separators.json @@ -0,0 +1,32 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "stringValue": "bindplane-otel-attributes-baba:you,bindplane-otel-attributes-host:\"myh,ost\",\"nam,e\":\"test:\",\"nested-n1:\":1,nested-n2:2,severity:155" + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/output/parsed-flattened-log-kv-pipe.json b/processor/marshalprocessor/testdata/output/parsed-flattened-log-kv-pipe.json new file mode 100644 index 000000000..e4940a98b --- /dev/null +++ b/processor/marshalprocessor/testdata/output/parsed-flattened-log-kv-pipe.json @@ -0,0 +1,32 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "stringValue": "bindplane-otel-attributes-baba=you|bindplane-otel-attributes-host=myhost|name=test|nested-n1=1|nested-n2=2|severity=155" + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/output/parsed-flattened-log-kv-plus.json b/processor/marshalprocessor/testdata/output/parsed-flattened-log-kv-plus.json new file mode 100644 index 000000000..0bb482a5f --- /dev/null +++ b/processor/marshalprocessor/testdata/output/parsed-flattened-log-kv-plus.json @@ -0,0 +1,32 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "stringValue": "bindplane-otel-attributes-baba+you bindplane-otel-attributes-host+myhost name+test nested-n1+1 nested-n2+2 severity+155" + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/output/parsed-flattened-log-kv-separators-and-quotes.json b/processor/marshalprocessor/testdata/output/parsed-flattened-log-kv-separators-and-quotes.json new file mode 100644 index 000000000..1cb64c112 --- /dev/null +++ b/processor/marshalprocessor/testdata/output/parsed-flattened-log-kv-separators-and-quotes.json @@ -0,0 +1,32 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "stringValue": "bindplane-otel-attributes-baba=you bindplane-otel-attributes-host=\"myh \\\"ost\" \"nam e\"=\"test=\" \"nested-n1=\"=1 nested-n2=2 s\\\"everity=155" + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/output/parsed-flattened-log-kv-separators.json b/processor/marshalprocessor/testdata/output/parsed-flattened-log-kv-separators.json new file mode 100644 index 000000000..6562d15fb --- /dev/null +++ b/processor/marshalprocessor/testdata/output/parsed-flattened-log-kv-separators.json @@ -0,0 +1,32 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "stringValue": "bindplane-otel-attributes-baba=you bindplane-otel-attributes-host=\"myh ost\" \"nam e\"=\"test=\" \"nested-n1=\"=1 nested-n2=2 severity=155" + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/output/parsed-flattened-log-kv.json b/processor/marshalprocessor/testdata/output/parsed-flattened-log-kv.json new file mode 100644 index 000000000..14a2a02a7 --- /dev/null +++ b/processor/marshalprocessor/testdata/output/parsed-flattened-log-kv.json @@ -0,0 +1,32 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "stringValue": "bindplane-otel-attributes-baba=you bindplane-otel-attributes-host=myhost name=test nested-n1=1 nested-n2=2 severity=155" + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/output/parsed-log-deeply-nested-kv-map.json b/processor/marshalprocessor/testdata/output/parsed-log-deeply-nested-kv-map.json new file mode 100644 index 000000000..dac5ba011 --- /dev/null +++ b/processor/marshalprocessor/testdata/output/parsed-log-deeply-nested-kv-map.json @@ -0,0 +1,32 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "stringValue": "bindplane-otel-attributes=[baba:you|host:myhost] name=test nested=[n1:1|n2:[n3:3|n4:4]] severity=155" + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/output/parsed-log-deeply-nested-kv.json b/processor/marshalprocessor/testdata/output/parsed-log-deeply-nested-kv.json new file mode 100644 index 000000000..817e82d6c --- /dev/null +++ b/processor/marshalprocessor/testdata/output/parsed-log-deeply-nested-kv.json @@ -0,0 +1,32 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "stringValue": "bindplane-otel-attributes=\"[baba=you,host=myhost]\" name=test nested=\"[n1=1,n2=[n3=3,n4=4]]\" severity=155" + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/output/parsed-log-deeply-nested-with-separators-kv-map-2.json b/processor/marshalprocessor/testdata/output/parsed-log-deeply-nested-with-separators-kv-map-2.json new file mode 100644 index 000000000..338b77031 --- /dev/null +++ b/processor/marshalprocessor/testdata/output/parsed-log-deeply-nested-with-separators-kv-map-2.json @@ -0,0 +1,32 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "stringValue": "bindplane-otel-attributes=[baba:you|host:myhost] name=test nested=[\\\"a not = great string\\\":1|n2:[n3:\\\"a not = great string\\\"|n4:4]] severity=155" + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/output/parsed-log-deeply-nested-with-separators-kv-map.json b/processor/marshalprocessor/testdata/output/parsed-log-deeply-nested-with-separators-kv-map.json new file mode 100644 index 000000000..9ea316166 --- /dev/null +++ b/processor/marshalprocessor/testdata/output/parsed-log-deeply-nested-with-separators-kv-map.json @@ -0,0 +1,32 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "stringValue": "bindplane-otel-attributes=[baba:you|host:myhost] name=test nested=[\\\"a not :|| []]great string \\\\\\\"\\\":1|n2:[n3:\\\"a not :|| []]great string \\\\\\\"\\\"|n4:4]] severity=155" + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/output/parsed-log-deeply-nested-with-separators-kv.json b/processor/marshalprocessor/testdata/output/parsed-log-deeply-nested-with-separators-kv.json new file mode 100644 index 000000000..c4cc51ec4 --- /dev/null +++ b/processor/marshalprocessor/testdata/output/parsed-log-deeply-nested-with-separators-kv.json @@ -0,0 +1,32 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "stringValue": "bindplane-otel-attributes=\"[baba=you,host=myhost]\" name=test nested=\"[\\\"a not =,, []]great string \\\\\\\"\\\"=1,n2=[n3=\\\"a not =,, []]great string \\\\\\\"\\\",n4=4]]\" severity=155" + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/output/parsed-log-json.json b/processor/marshalprocessor/testdata/output/parsed-log-json.json new file mode 100644 index 000000000..a412bca4d --- /dev/null +++ b/processor/marshalprocessor/testdata/output/parsed-log-json.json @@ -0,0 +1,32 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "stringValue": "{\"bindplane-otel-attributes\":{\"baba\":\"you\",\"host\":\"myhost\"},\"name\":\"test\",\"nested\":{\"n1\":1,\"n2\":2},\"severity\":155}" + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/output/parsed-log-kv.json b/processor/marshalprocessor/testdata/output/parsed-log-kv.json new file mode 100644 index 000000000..b1ee5b461 --- /dev/null +++ b/processor/marshalprocessor/testdata/output/parsed-log-kv.json @@ -0,0 +1,32 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "stringValue": "bindplane-otel-attributes=\"[baba=you,host=myhost]\" name=test nested=\"[n1=1,n2=2]\" severity=155" + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/output/parsed-nested-log-with-slice-kv.json b/processor/marshalprocessor/testdata/output/parsed-nested-log-with-slice-kv.json new file mode 100644 index 000000000..500582914 --- /dev/null +++ b/processor/marshalprocessor/testdata/output/parsed-nested-log-with-slice-kv.json @@ -0,0 +1,32 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "stringValue": "bindplane-otel-attributes=\"[baba=you,host=myhost]\" name=test nested=\"[n1=\\\"[1,2,[1,2],{\\\\\\\"n1\\\\\\\":1,\\\\\\\"n2\\\\\\\":2}]\\\",n2=2]\" severity=155" + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/processor/marshalprocessor/testdata/output/string-log.json b/processor/marshalprocessor/testdata/output/string-log.json new file mode 100644 index 000000000..796158045 --- /dev/null +++ b/processor/marshalprocessor/testdata/output/string-log.json @@ -0,0 +1,32 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { "key": "field1", "value": { "stringValue": "val1" } }, + { "key": "field2", "value": { "stringValue": "val2" } } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1724942726840056000", + "severityNumber": 5, + "body": { + "stringValue": "this log is a string that cannot be parsed" + }, + "attributes": [ + { "key": "baba", "value": { "stringValue": "you" } }, + { "key": "host", "value": { "stringValue": "myhost" } } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +}