Skip to content

Commit

Permalink
[pkg/telemetryquerylanguage] add replace_key_all_patterns function (#…
Browse files Browse the repository at this point in the history 
…12991)

* [pkg/telemetryquerylanguage] enhance replace_all_patterns function
  • Loading branch information
@fatsheep9146
fatsheep9146 authored Oct 6, 2022
1 parent e80df4f commit 2ba332a
Show file tree
Hide file tree
Showing 7 changed files with 193 additions and 18 deletions.
16 changes: 16 additions & 0 deletions .chloggen/add-replace-key-all-patterns.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
change_type: breaking

# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
component: pkg/ottl

# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
note: "Enhance replace_all_patterns functions to take a new parameter that specifies whether the function applies to keys or values."

# One or more tracking issues related to the change
issues: [12631]

# (Optional) One or more lines of additional information to render under the primary note.
# These lines will be padded with 2 spaces and then inserted directly into the document.
# Use pipe (|) for multiline entries.
subtext:
12 changes: 8 additions & 4 deletions pkg/ottl/ottlfuncs/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,17 +213,20 @@ Examples:

## replace_all_patterns

`replace_all_patterns(target, regex, replacement)`
`replace_all_patterns(target, mode, regex, replacement)`

The `replace_all_patterns` function replaces any segments in a string value that match the regex pattern with the replacement string.
The `replace_all_patterns` function replaces any segments in a string value or key that match the regex pattern with the replacement string.

`target` is a path expression to a `pdata.Map` type field. `regex` is a regex string indicating a segment to replace. `replacement` is a string.

`mode` determines whether the match and replace will occur on the map's value or key. Valid values are `key` and `value`.

If one or more sections of `target` match `regex` they will get replaced with `replacement`.

Examples:

- `replace_all_patterns(attributes, "/account/\\d{4}", "/account/{accountId}")`
- `replace_all_patterns(attributes, "value", "/account/\\d{4}", "/account/{accountId}")`
- `replace_all_patterns(attributes, "key", "/account/\\d{4}", "/account/{accountId}")`

## replace_pattern

Expand Down Expand Up @@ -292,4 +295,5 @@ Examples:
- `truncate_all(attributes, 100)`


- `truncate_all(resource.attributes, 50)`
- `truncate_all(resource.attributes, 50)`

35 changes: 28 additions & 7 deletions pkg/ottl/ottlfuncs/func_replace_all_patterns.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,20 @@ import (
"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl"
)

func ReplaceAllPatterns[K any](target ottl.GetSetter[K], regexPattern string, replacement string) (ottl.ExprFunc[K], error) {
const (
modeKey = "key"
modeValue = "value"
)

func ReplaceAllPatterns[K any](target ottl.GetSetter[K], mode string, regexPattern string, replacement string) (ottl.ExprFunc[K], error) {
compiledPattern, err := regexp.Compile(regexPattern)
if err != nil {
return nil, fmt.Errorf("the regex pattern supplied to replace_all_patterns is not a valid pattern: %w", err)
}
if mode != modeValue && mode != modeKey {
return nil, fmt.Errorf("invalid mode %v, must be either 'key' or 'value'", mode)
}

return func(ctx K) interface{} {
val := target.Get(ctx)
if val == nil {
Expand All @@ -37,17 +46,29 @@ func ReplaceAllPatterns[K any](target ottl.GetSetter[K], regexPattern string, re
if !ok {
return nil
}

updated := pcommon.NewMap()
attrs.CopyTo(updated)
updated.Range(func(key string, value pcommon.Value) bool {
stringVal := value.Str()
if compiledPattern.MatchString(stringVal) {
value.SetStr(compiledPattern.ReplaceAllLiteralString(stringVal, replacement))
updated.EnsureCapacity(attrs.Len())
attrs.Range(func(key string, originalValue pcommon.Value) bool {
switch mode {
case modeValue:
if compiledPattern.MatchString(originalValue.Str()) {
updatedString := compiledPattern.ReplaceAllLiteralString(originalValue.Str(), replacement)
updated.PutString(key, updatedString)
} else {
updated.PutString(key, originalValue.Str())
}
case modeKey:
if compiledPattern.MatchString(key) {
updatedKey := compiledPattern.ReplaceAllLiteralString(key, replacement)
updated.PutString(updatedKey, originalValue.Str())
} else {
updated.PutString(key, originalValue.Str())
}
}
return true
})
target.Set(ctx, updated)

return nil
}, nil
}
68 changes: 64 additions & 4 deletions pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,15 @@ func Test_replaceAllPatterns(t *testing.T) {
tests := []struct {
name string
target ottl.GetSetter[pcommon.Map]
mode string
pattern string
replacement string
want func(pcommon.Map)
}{
{
name: "replace only matches",
target: target,
mode: modeValue,
pattern: "hello",
replacement: "hello {universe}",
want: func(expectedMap pcommon.Map) {
Expand All @@ -60,6 +62,7 @@ func Test_replaceAllPatterns(t *testing.T) {
{
name: "no matches",
target: target,
mode: modeValue,
pattern: "nothing",
replacement: "nothing {matches}",
want: func(expectedMap pcommon.Map) {
Expand All @@ -71,6 +74,7 @@ func Test_replaceAllPatterns(t *testing.T) {
{
name: "multiple regex match",
target: target,
mode: modeValue,
pattern: `world[^\s]*(\s?)`,
replacement: "**** ",
want: func(expectedMap pcommon.Map) {
Expand All @@ -79,13 +83,52 @@ func Test_replaceAllPatterns(t *testing.T) {
expectedMap.PutString("test3", "goodbye **** and **** ")
},
},
{
name: "replace only matches",
target: target,
mode: modeKey,
pattern: "test2",
replacement: "foo",
want: func(expectedMap pcommon.Map) {
expectedMap.Clear()
expectedMap.PutString("test", "hello world")
expectedMap.PutString("foo", "hello")
expectedMap.PutString("test3", "goodbye world1 and world2")
},
},
{
name: "no matches",
target: target,
mode: modeKey,
pattern: "nothing",
replacement: "nothing {matches}",
want: func(expectedMap pcommon.Map) {
expectedMap.Clear()
expectedMap.PutString("test", "hello world")
expectedMap.PutString("test2", "hello")
expectedMap.PutString("test3", "goodbye world1 and world2")
},
},
{
name: "multiple regex match",
target: target,
mode: modeKey,
pattern: `test`,
replacement: "test.",
want: func(expectedMap pcommon.Map) {
expectedMap.Clear()
expectedMap.PutString("test.", "hello world")
expectedMap.PutString("test.2", "hello")
expectedMap.PutString("test.3", "goodbye world1 and world2")
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
scenarioMap := pcommon.NewMap()
input.CopyTo(scenarioMap)

exprFunc, err := ReplaceAllPatterns[pcommon.Map](tt.target, tt.pattern, tt.replacement)
exprFunc, err := ReplaceAllPatterns[pcommon.Map](tt.target, tt.mode, tt.pattern, tt.replacement)
require.NoError(t, err)
exprFunc(scenarioMap)

Expand All @@ -109,7 +152,7 @@ func Test_replaceAllPatterns_bad_input(t *testing.T) {
},
}

exprFunc, err := ReplaceAllPatterns[interface{}](target, "regexpattern", "{replacement}")
exprFunc, err := ReplaceAllPatterns[interface{}](target, modeValue, "regexpattern", "{replacement}")
assert.Nil(t, err)

exprFunc(input)
Expand All @@ -127,7 +170,7 @@ func Test_replaceAllPatterns_get_nil(t *testing.T) {
},
}

exprFunc, err := ReplaceAllPatterns[interface{}](target, "regexp", "{anything}")
exprFunc, err := ReplaceAllPatterns[interface{}](target, modeValue, "regexp", "{anything}")
require.NoError(t, err)
exprFunc(nil)
}
Expand All @@ -144,8 +187,25 @@ func Test_replaceAllPatterns_invalid_pattern(t *testing.T) {
}

invalidRegexPattern := "*"
exprFunc, err := ReplaceAllPatterns[interface{}](target, invalidRegexPattern, "{anything}")
exprFunc, err := ReplaceAllPatterns[interface{}](target, modeValue, invalidRegexPattern, "{anything}")
require.Error(t, err)
assert.ErrorContains(t, err, "error parsing regexp:")
assert.Nil(t, exprFunc)
}

func Test_replaceAllPatterns_invalid_model(t *testing.T) {
target := &ottl.StandardGetSetter[interface{}]{
Getter: func(ctx interface{}) interface{} {
t.Errorf("nothing should be received in this scenario")
return nil
},
Setter: func(ctx interface{}, val interface{}) {
t.Errorf("nothing should be set in this scenario")
},
}

invalidMode := "invalid"
exprFunc, err := ReplaceAllPatterns[interface{}](target, invalidMode, "regex", "{anything}")
assert.Nil(t, exprFunc)
assert.Contains(t, err.Error(), "invalid mode")
}
17 changes: 16 additions & 1 deletion processor/transformprocessor/internal/logs/processor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,12 +77,27 @@ func TestProcess(t *testing.T) {
},
},
{
statement: `replace_all_patterns(attributes, "get", "post")`,
statement: `replace_all_patterns(attributes, "value", "get", "post")`,
want: func(td plog.Logs) {
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().PutString("http.method", "post")
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(1).Attributes().PutString("http.method", "post")
},
},
{
statement: `replace_all_patterns(attributes, "key", "http.url", "url")`,
want: func(td plog.Logs) {
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().Clear()
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().PutString("http.method", "get")
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().PutString("http.path", "/health")
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().PutString("url", "http://localhost/health")
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().PutString("flags", "A|B|C")
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(1).Attributes().Clear()
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(1).Attributes().PutString("http.method", "get")
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(1).Attributes().PutString("http.path", "/health")
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(1).Attributes().PutString("url", "http://localhost/health")
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(1).Attributes().PutString("flags", "C|D")
},
},
{
statement: `set(attributes["test"], "pass") where dropped_attributes_count == 1`,
want: func(td plog.Logs) {
Expand Down
45 changes: 44 additions & 1 deletion processor/transformprocessor/internal/metrics/processor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ func TestProcess(t *testing.T) {
},
},
{
statements: []string{`replace_all_patterns(attributes, "test1", "pass")`},
statements: []string{`replace_all_patterns(attributes, "value", "test1", "pass")`},
want: func(td pmetric.Metrics) {
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().PutString("attr1", "pass")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().PutString("attr1", "pass")
Expand All @@ -173,6 +173,49 @@ func TestProcess(t *testing.T) {
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(3).Summary().DataPoints().At(0).Attributes().PutString("attr1", "pass")
},
},
{
statements: []string{`replace_all_patterns(attributes, "key", "attr3", "attr4")`},
want: func(td pmetric.Metrics) {
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().Clear()
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().PutString("attr1", "test1")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().PutString("attr2", "test2")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().PutString("attr4", "test3")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().PutString("flags", "A|B|C")

td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().Clear()
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().PutString("attr1", "test1")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().PutString("attr2", "test2")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().PutString("attr4", "test3")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().PutString("flags", "A|B|C")

td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().At(0).Attributes().Clear()
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().At(0).Attributes().PutString("attr1", "test1")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().At(0).Attributes().PutString("attr2", "test2")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().At(0).Attributes().PutString("attr4", "test3")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().At(0).Attributes().PutString("flags", "C|D")

td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().At(1).Attributes().Clear()
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().At(1).Attributes().PutString("attr1", "test1")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().At(1).Attributes().PutString("attr2", "test2")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().At(1).Attributes().PutString("attr4", "test3")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().At(1).Attributes().PutString("flags", "C|D")

td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(2).ExponentialHistogram().DataPoints().At(0).Attributes().Clear()
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(2).ExponentialHistogram().DataPoints().At(0).Attributes().PutString("attr1", "test1")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(2).ExponentialHistogram().DataPoints().At(0).Attributes().PutString("attr2", "test2")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(2).ExponentialHistogram().DataPoints().At(0).Attributes().PutString("attr4", "test3")

td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(2).ExponentialHistogram().DataPoints().At(1).Attributes().Clear()
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(2).ExponentialHistogram().DataPoints().At(1).Attributes().PutString("attr1", "test1")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(2).ExponentialHistogram().DataPoints().At(1).Attributes().PutString("attr2", "test2")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(2).ExponentialHistogram().DataPoints().At(1).Attributes().PutString("attr4", "test3")

td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(3).Summary().DataPoints().At(0).Attributes().Clear()
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(3).Summary().DataPoints().At(0).Attributes().PutString("attr1", "test1")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(3).Summary().DataPoints().At(0).Attributes().PutString("attr2", "test2")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(3).Summary().DataPoints().At(0).Attributes().PutString("attr4", "test3")
},
},
{
statements: []string{`convert_summary_count_val_to_sum("delta", true) where metric.name == "operationD"`},
want: func(td pmetric.Metrics) {
Expand Down
18 changes: 17 additions & 1 deletion processor/transformprocessor/internal/traces/processor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,12 +119,28 @@ func TestProcess(t *testing.T) {
},
},
{
statement: `replace_all_patterns(attributes, "get", "post")`,
statement: `replace_all_patterns(attributes, "value", "get", "post")`,
want: func(td ptrace.Traces) {
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().PutString("http.method", "post")
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(1).Attributes().PutString("http.method", "post")
},
},
{
statement: `replace_all_patterns(attributes, "key", "http.url", "url")`,
want: func(td ptrace.Traces) {
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().Clear()
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().PutString("http.method", "get")
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().PutString("http.path", "/health")
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().PutString("url", "http://localhost/health")
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().PutString("flags", "A|B|C")

td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(1).Attributes().Clear()
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(1).Attributes().PutString("http.method", "get")
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(1).Attributes().PutString("http.path", "/health")
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(1).Attributes().PutString("url", "http://localhost/health")
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(1).Attributes().PutString("flags", "C|D")
},
},
{
statement: `set(attributes["test"], "pass") where IsMatch(name, "operation[AC]") == true`,
want: func(td ptrace.Traces) {
Expand Down

0 comments on commit 2ba332a

Please sign in to comment.