From 336226dd6575b289d631cb06e88f98f4bf0ed64c Mon Sep 17 00:00:00 2001 From: Raj Nishtala Date: Mon, 16 Oct 2023 12:54:24 -0400 Subject: [PATCH 1/2] feat(pkg/ottl) Adding an optional replacement prefix argument to the replace_pattern editors --- .chloggen/ottl-replace-pattern.yaml | 3 +- pkg/ottl/ottlfuncs/README.md | 13 ++-- .../ottlfuncs/func_replace_all_patterns.go | 21 ++++-- .../func_replace_all_patterns_test.go | 65 ++++++++++++------- pkg/ottl/ottlfuncs/func_replace_pattern.go | 16 +++-- .../ottlfuncs/func_replace_pattern_test.go | 52 +++++++++------ 6 files changed, 106 insertions(+), 64 deletions(-) diff --git a/.chloggen/ottl-replace-pattern.yaml b/.chloggen/ottl-replace-pattern.yaml index 4e85584d1439..c8778c080422 100755 --- a/.chloggen/ottl-replace-pattern.yaml +++ b/.chloggen/ottl-replace-pattern.yaml @@ -7,7 +7,7 @@ change_type: enhancement component: pkg/ottl # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). -note: Add optional Converter parameters to replacement Editors +note: Add optional parameters to replacement Editors # Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. issues: [27235] @@ -21,6 +21,7 @@ subtext: | - `replace_all_patterns` - `replace_match` - `replace_all_matches` + An optional replacement prefix argument can also be passed to `replace_pattern` and `replace_all_patterns` to be prepended to the replacement string. # If your change doesn't affect end users or the exported elements of any package, # you should instead start your pull request title with [chore] or use the "Skip Changelog" label. diff --git a/pkg/ottl/ottlfuncs/README.md b/pkg/ottl/ottlfuncs/README.md index 47c7fc7460de..9491d4daf053 100644 --- a/pkg/ottl/ottlfuncs/README.md +++ b/pkg/ottl/ottlfuncs/README.md @@ -165,7 +165,7 @@ Examples: ### replace_all_patterns -`replace_all_patterns(target, mode, regex, replacement, function)` +`replace_all_patterns(target, mode, regex, replacement, replacementPrefix, function)` The `replace_all_patterns` function replaces any segments in a string value or key that match the regex pattern with the replacement string. @@ -175,7 +175,7 @@ The `replace_all_patterns` function replaces any segments in a string value or k If one or more sections of `target` match `regex` they will get replaced with `replacement`. -The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). +The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). `replacementPrefix` is an optional string argument to prefix a `replacement` string with. The `function` is an optional argument that can take in any Converter that accepts a (`replacement`) string and returns a string. An example is a hash function that replaces any matching regex pattern with the hash value of `replacement`. @@ -188,7 +188,8 @@ Examples: - `replace_all_patterns(attributes, "value", "/account/\\d{4}", "/account/{accountId}")` - `replace_all_patterns(attributes, "key", "/account/\\d{4}", "/account/{accountId}")` - `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "k8s.$$1.")` -- `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "k8s.$$1.", SHA256)` +- `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "$$1.", "k8s.)` +- `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "$$1.", "k8s.", SHA256)` Note that when using OTTL within the collector's configuration file, `$` must be escaped to `$$` to bypass environment variable substitution logic. To input a literal `$` from the configuration file, use `$$$`. @@ -216,7 +217,7 @@ Examples: ### replace_pattern -`replace_pattern(target, regex, replacement, function)` +`replace_pattern(target, regex, replacement, replacementPrefix, function)` The `replace_pattern` function allows replacing all string sections that match a regex pattern with a new value. @@ -224,7 +225,7 @@ The `replace_pattern` function allows replacing all string sections that match a If one or more sections of `target` match `regex` they will get replaced with `replacement`. -The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). +The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). `replacementPrefix` is an optional string argument to prefix a `replacement` string with. The `function` is an optional argument that can take in any Converter that accepts a (`replacement`) string and returns a string. An example is a hash function that replaces a matching regex pattern with the hash value of `replacement`. @@ -236,7 +237,7 @@ Examples: - `replace_pattern(resource.attributes["process.command_line"], "password\\=[^\\s]*(\\s?)", "password=***")` - `replace_pattern(name, "^kube_([0-9A-Za-z]+_)", "k8s.$$1.")` -- `replace_pattern(name, "^kube_([0-9A-Za-z]+_)", "k8s.$$1.", SHA256)` +- `replace_pattern(name, "^kube_([0-9A-Za-z]+_)", "$$1.", "k8s.", SHA256)` Note that when using OTTL within the collector's configuration file, `$` must be escaped to `$$` to bypass environment variable substitution logic. To input a literal `$` from the configuration file, use `$$$`. diff --git a/pkg/ottl/ottlfuncs/func_replace_all_patterns.go b/pkg/ottl/ottlfuncs/func_replace_all_patterns.go index 47021b6bff01..175be5e0b197 100644 --- a/pkg/ottl/ottlfuncs/func_replace_all_patterns.go +++ b/pkg/ottl/ottlfuncs/func_replace_all_patterns.go @@ -19,11 +19,12 @@ const ( ) type ReplaceAllPatternsArguments[K any] struct { - Target ottl.PMapGetter[K] - Mode string - RegexPattern string - Replacement ottl.StringGetter[K] - Function ottl.Optional[ottl.FunctionGetter[K]] + Target ottl.PMapGetter[K] + Mode string + RegexPattern string + Replacement ottl.StringGetter[K] + ReplacementPrefix ottl.Optional[string] + Function ottl.Optional[ottl.FunctionGetter[K]] } type replaceAllPatternFuncArgs[K any] struct { @@ -41,10 +42,10 @@ func createReplaceAllPatternsFunction[K any](_ ottl.FunctionContext, oArgs ottl. return nil, fmt.Errorf("ReplaceAllPatternsFactory args must be of type *ReplaceAllPatternsArguments[K]") } - return replaceAllPatterns(args.Target, args.Mode, args.RegexPattern, args.Replacement, args.Function) + return replaceAllPatterns(args.Target, args.Mode, args.RegexPattern, args.Replacement, args.ReplacementPrefix, args.Function) } -func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) { +func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPattern string, replacement ottl.StringGetter[K], replacementPrefix ottl.Optional[string], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) { compiledPattern, err := regexp.Compile(regexPattern) if err != nil { return nil, fmt.Errorf("the regex pattern supplied to replace_all_patterns is not a valid pattern: %w", err) @@ -86,6 +87,9 @@ func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPatt switch mode { case modeValue: if compiledPattern.MatchString(originalValue.Str()) { + if !replacementPrefix.IsEmpty() { // If replacementPrefix is not empty, add it to the replacement value + replacementVal = replacementPrefix.Get() + replacementVal + } updatedString := compiledPattern.ReplaceAllString(originalValue.Str(), replacementVal) updated.PutStr(key, updatedString) } else { @@ -93,6 +97,9 @@ func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPatt } case modeKey: if compiledPattern.MatchString(key) { + if !replacementPrefix.IsEmpty() { // If replacementPrefix is not empty, add it to the replacement value + replacementVal = replacementPrefix.Get() + replacementVal + } updatedKey := compiledPattern.ReplaceAllString(key, replacementVal) originalValue.CopyTo(updated.PutEmpty(updatedKey)) } else { diff --git a/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go b/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go index 29bb2b2669f0..7ea56dc8d5e3 100644 --- a/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go @@ -39,13 +39,14 @@ func Test_replaceAllPatterns(t *testing.T) { } tests := []struct { - name string - target ottl.PMapGetter[pcommon.Map] - mode string - pattern string - replacement ottl.StringGetter[pcommon.Map] - function ottl.Optional[ottl.FunctionGetter[pcommon.Map]] - want func(pcommon.Map) + name string + target ottl.PMapGetter[pcommon.Map] + mode string + pattern string + replacement ottl.StringGetter[pcommon.Map] + replacementPrefix ottl.Optional[string] + function ottl.Optional[ottl.FunctionGetter[pcommon.Map]] + want func(pcommon.Map) }{ { name: "replace only matches (with hash function)", @@ -57,7 +58,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "hello {universe}", nil }, }, - function: optionalArg, + replacementPrefix: ottl.Optional[string]{}, + function: optionalArg, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "4804d6b7f03268e33f78c484977f3d81771220df07cc6aac4ad4868102141fad world") expectedMap.PutStr("test2", "4804d6b7f03268e33f78c484977f3d81771220df07cc6aac4ad4868102141fad") @@ -77,7 +79,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "hello {universe}", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello {universe} world") expectedMap.PutStr("test2", "hello {universe}") @@ -97,7 +100,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "nothing {matches}", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello world") expectedMap.PutStr("test2", "hello") @@ -117,7 +121,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "**** ", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello **** ") expectedMap.PutStr("test2", "hello") @@ -137,7 +142,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "foo", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() expectedMap.PutStr("test", "hello world") @@ -158,7 +164,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "nothing {matches}", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() expectedMap.PutStr("test", "hello world") @@ -179,7 +186,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "test.", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() expectedMap.PutStr("test.", "hello world") @@ -200,7 +208,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "world-$1", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() expectedMap.PutStr("test", "hello world") @@ -221,7 +230,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "test-$1", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello world") expectedMap.PutStr("test-2", "hello") @@ -241,7 +251,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "$$world-$1", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() expectedMap.PutStr("test", "hello world") @@ -258,7 +269,7 @@ func Test_replaceAllPatterns(t *testing.T) { scenarioMap := pcommon.NewMap() input.CopyTo(scenarioMap) - exprFunc, err := replaceAllPatterns[pcommon.Map](tt.target, tt.mode, tt.pattern, tt.replacement, tt.function) + exprFunc, err := replaceAllPatterns[pcommon.Map](tt.target, tt.mode, tt.pattern, tt.replacement, tt.replacementPrefix, tt.function) assert.NoError(t, err) _, err = exprFunc(nil, scenarioMap) @@ -284,9 +295,10 @@ func Test_replaceAllPatterns_bad_input(t *testing.T) { return "{replacement}", nil }, } + replacementPrefix := ottl.Optional[string]{} function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} - exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, "regexpattern", replacement, function) + exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, "regexpattern", replacement, replacementPrefix, function) assert.Nil(t, err) _, err = exprFunc(nil, input) @@ -305,9 +317,10 @@ func Test_replaceAllPatterns_bad_function_input(t *testing.T) { return nil, nil }, } + replacementPrefix := ottl.Optional[string]{} function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} - exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, "regexp", replacement, function) + exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, "regexp", replacement, replacementPrefix, function) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -334,9 +347,10 @@ func Test_replaceAllPatterns_bad_function_result(t *testing.T) { }, Fact: StandardConverters[interface{}]()["IsString"], } + replacementPrefix := ottl.Optional[string]{} function := ottl.NewTestingOptional[ottl.FunctionGetter[interface{}]](ottlValue) - exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, "regexp", replacement, function) + exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, "regexp", replacement, replacementPrefix, function) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -355,9 +369,10 @@ func Test_replaceAllPatterns_get_nil(t *testing.T) { return "{anything}", nil }, } + replacementPrefix := ottl.Optional[string]{} function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} - exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, "regexp", replacement, function) + exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, "regexp", replacement, replacementPrefix, function) assert.NoError(t, err) _, err = exprFunc(nil, nil) @@ -376,10 +391,11 @@ func Test_replaceAllPatterns_invalid_pattern(t *testing.T) { return "{anything}", nil }, } + replacementPrefix := ottl.Optional[string]{} function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} invalidRegexPattern := "*" - exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, invalidRegexPattern, replacement, function) + exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, invalidRegexPattern, replacement, replacementPrefix, function) require.Error(t, err) assert.ErrorContains(t, err, "error parsing regexp:") assert.Nil(t, exprFunc) @@ -397,10 +413,11 @@ func Test_replaceAllPatterns_invalid_model(t *testing.T) { return "{anything}", nil }, } + replacementPrefix := ottl.Optional[string]{} function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} invalidMode := "invalid" - exprFunc, err := replaceAllPatterns[interface{}](target, invalidMode, "regex", replacement, function) + exprFunc, err := replaceAllPatterns[interface{}](target, invalidMode, "regex", replacement, replacementPrefix, function) assert.Nil(t, exprFunc) assert.Contains(t, err.Error(), "invalid mode") } diff --git a/pkg/ottl/ottlfuncs/func_replace_pattern.go b/pkg/ottl/ottlfuncs/func_replace_pattern.go index b69ea098e2fa..48ad1007269a 100644 --- a/pkg/ottl/ottlfuncs/func_replace_pattern.go +++ b/pkg/ottl/ottlfuncs/func_replace_pattern.go @@ -12,10 +12,11 @@ import ( ) type ReplacePatternArguments[K any] struct { - Target ottl.GetSetter[K] - RegexPattern string - Replacement ottl.StringGetter[K] - Function ottl.Optional[ottl.FunctionGetter[K]] + Target ottl.GetSetter[K] + RegexPattern string + Replacement ottl.StringGetter[K] + ReplacementPrefix ottl.Optional[string] // ReplacementPrefix is an optional prefix to add to the replacement value + Function ottl.Optional[ottl.FunctionGetter[K]] } type replacePatternFuncArgs[K any] struct { @@ -33,10 +34,10 @@ func createReplacePatternFunction[K any](_ ottl.FunctionContext, oArgs ottl.Argu return nil, fmt.Errorf("ReplacePatternFactory args must be of type *ReplacePatternArguments[K]") } - return replacePattern(args.Target, args.RegexPattern, args.Replacement, args.Function) + return replacePattern(args.Target, args.RegexPattern, args.Replacement, args.ReplacementPrefix, args.Function) } -func replacePattern[K any](target ottl.GetSetter[K], regexPattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) { +func replacePattern[K any](target ottl.GetSetter[K], regexPattern string, replacement ottl.StringGetter[K], replacementPrefix ottl.Optional[string], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) { compiledPattern, err := regexp.Compile(regexPattern) if err != nil { return nil, fmt.Errorf("the regex pattern supplied to replace_pattern is not a valid pattern: %w", err) @@ -73,6 +74,9 @@ func replacePattern[K any](target ottl.GetSetter[K], regexPattern string, replac } if originalValStr, ok := originalVal.(string); ok { if compiledPattern.MatchString(originalValStr) { + if !replacementPrefix.IsEmpty() { // If replacementPrefix is not empty, add it to the replacement value + replacementVal = replacementPrefix.Get() + replacementVal + } updatedStr := compiledPattern.ReplaceAllString(originalValStr, replacementVal) err = target.Set(ctx, tCtx, updatedStr) if err != nil { diff --git a/pkg/ottl/ottlfuncs/func_replace_pattern_test.go b/pkg/ottl/ottlfuncs/func_replace_pattern_test.go index 87812b2684ac..54ad73798be5 100644 --- a/pkg/ottl/ottlfuncs/func_replace_pattern_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_pattern_test.go @@ -35,25 +35,27 @@ func Test_replacePattern(t *testing.T) { } tests := []struct { - name string - target ottl.GetSetter[pcommon.Value] - pattern string - replacement ottl.StringGetter[pcommon.Value] - function ottl.Optional[ottl.FunctionGetter[pcommon.Value]] - want func(pcommon.Value) + name string + target ottl.GetSetter[pcommon.Value] + pattern string + replacement ottl.StringGetter[pcommon.Value] + replacementPrefix ottl.Optional[string] + function ottl.Optional[ottl.FunctionGetter[pcommon.Value]] + want func(pcommon.Value) }{ { name: "replace regex match (with hash function)", target: target, - pattern: `passwd\=[^\s]*(\s?)`, + pattern: `passwd\=[^\s]*`, replacement: ottl.StandardStringGetter[pcommon.Value]{ Getter: func(context.Context, pcommon.Value) (interface{}, error) { return "passwd=*** ", nil }, }, - function: optionalArg, + replacementPrefix: ottl.NewTestingOptional[string]("passwd="), + function: optionalArg, want: func(expectedValue pcommon.Value) { - expectedValue.SetStr("application 0f2407f2d83337b1f757eb1754a7643ce0e8fba620bc605c54566cd6dfd838beotherarg=notsensitive key1 key2") + expectedValue.SetStr("application passwd=0f2407f2d83337b1f757eb1754a7643ce0e8fba620bc605c54566cd6dfd838be otherarg=notsensitive key1 key2") }, }, { @@ -65,7 +67,8 @@ func Test_replacePattern(t *testing.T) { return "passwd=*** ", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=*** otherarg=notsensitive key1 key2") }, @@ -79,7 +82,8 @@ func Test_replacePattern(t *testing.T) { return "shouldnotbeinoutput", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=sensitivedtata otherarg=notsensitive key1 key2") }, @@ -93,7 +97,8 @@ func Test_replacePattern(t *testing.T) { return "**** ", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=sensitivedtata otherarg=notsensitive **** **** ") }, @@ -107,7 +112,8 @@ func Test_replacePattern(t *testing.T) { return "$1:$2", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd:sensitivedtata otherarg:notsensitive key1 key2") }, @@ -121,7 +127,8 @@ func Test_replacePattern(t *testing.T) { return "passwd=$$$$$$ ", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=$$$ otherarg=notsensitive key1 key2") }, @@ -130,7 +137,7 @@ func Test_replacePattern(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { scenarioValue := pcommon.NewValueStr(input.Str()) - exprFunc, err := replacePattern(tt.target, tt.pattern, tt.replacement, tt.function) + exprFunc, err := replacePattern(tt.target, tt.pattern, tt.replacement, tt.replacementPrefix, tt.function) assert.NoError(t, err) result, err := exprFunc(nil, scenarioValue) @@ -161,9 +168,10 @@ func Test_replacePattern_bad_input(t *testing.T) { return "{replacement}", nil }, } + replacementPrefix := ottl.Optional[string]{} function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} - exprFunc, err := replacePattern[interface{}](target, "regexp", replacement, function) + exprFunc, err := replacePattern[interface{}](target, "regexp", replacement, replacementPrefix, function) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -188,9 +196,10 @@ func Test_replacePattern_bad_function_input(t *testing.T) { return nil, nil }, } + replacementPrefix := ottl.Optional[string]{} function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} - exprFunc, err := replacePattern[interface{}](target, "regexp", replacement, function) + exprFunc, err := replacePattern[interface{}](target, "regexp", replacement, replacementPrefix, function) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -221,9 +230,10 @@ func Test_replacePattern_bad_function_result(t *testing.T) { }, Fact: StandardConverters[interface{}]()["IsString"], } + replacementPrefix := ottl.Optional[string]{} function := ottl.NewTestingOptional[ottl.FunctionGetter[interface{}]](ottlValue) - exprFunc, err := replacePattern[interface{}](target, "regexp", replacement, function) + exprFunc, err := replacePattern[interface{}](target, "regexp", replacement, replacementPrefix, function) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -247,9 +257,10 @@ func Test_replacePattern_get_nil(t *testing.T) { return "{anything}", nil }, } + replacementPrefix := ottl.Optional[string]{} function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} - exprFunc, err := replacePattern[interface{}](target, `nomatch\=[^\s]*(\s?)`, replacement, function) + exprFunc, err := replacePattern[interface{}](target, `nomatch\=[^\s]*(\s?)`, replacement, replacementPrefix, function) assert.NoError(t, err) result, err := exprFunc(nil, nil) @@ -273,10 +284,11 @@ func Test_replacePatterns_invalid_pattern(t *testing.T) { return "{anything}", nil }, } + replacementPrefix := ottl.Optional[string]{} function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} invalidRegexPattern := "*" - _, err := replacePattern[interface{}](target, invalidRegexPattern, replacement, function) + _, err := replacePattern[interface{}](target, invalidRegexPattern, replacement, replacementPrefix, function) require.Error(t, err) assert.ErrorContains(t, err, "error parsing regexp:") } From ee00bae6fe4ce4519dcce0b6660bf8f7167a3096 Mon Sep 17 00:00:00 2001 From: Raj Nishtala Date: Wed, 18 Oct 2023 17:39:44 -0400 Subject: [PATCH 2/2] Change the replacement prefix string to be a format string, this allows for suffixes --- pkg/ottl/ottlfuncs/README.md | 14 +-- .../ottlfuncs/func_replace_all_patterns.go | 23 +++-- .../func_replace_all_patterns_test.go | 86 +++++++++++++------ pkg/ottl/ottlfuncs/func_replace_pattern.go | 15 ++-- .../ottlfuncs/func_replace_pattern_test.go | 85 ++++++++++++++---- 5 files changed, 162 insertions(+), 61 deletions(-) diff --git a/pkg/ottl/ottlfuncs/README.md b/pkg/ottl/ottlfuncs/README.md index 9491d4daf053..2acfddad9940 100644 --- a/pkg/ottl/ottlfuncs/README.md +++ b/pkg/ottl/ottlfuncs/README.md @@ -165,7 +165,7 @@ Examples: ### replace_all_patterns -`replace_all_patterns(target, mode, regex, replacement, replacementPrefix, function)` +`replace_all_patterns(target, mode, regex, replacement, replacementFormat, function)` The `replace_all_patterns` function replaces any segments in a string value or key that match the regex pattern with the replacement string. @@ -175,7 +175,7 @@ The `replace_all_patterns` function replaces any segments in a string value or k If one or more sections of `target` match `regex` they will get replaced with `replacement`. -The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). `replacementPrefix` is an optional string argument to prefix a `replacement` string with. +The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). `replacementFormat` is an optional string argument that specifies the format of the replacement. The `function` is an optional argument that can take in any Converter that accepts a (`replacement`) string and returns a string. An example is a hash function that replaces any matching regex pattern with the hash value of `replacement`. @@ -188,8 +188,8 @@ Examples: - `replace_all_patterns(attributes, "value", "/account/\\d{4}", "/account/{accountId}")` - `replace_all_patterns(attributes, "key", "/account/\\d{4}", "/account/{accountId}")` - `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "k8s.$$1.")` -- `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "$$1.", "k8s.)` -- `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "$$1.", "k8s.", SHA256)` +- `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "$$1.", "k8s.%s")` +- `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "$$1.", "k8s.%s", SHA256)` Note that when using OTTL within the collector's configuration file, `$` must be escaped to `$$` to bypass environment variable substitution logic. To input a literal `$` from the configuration file, use `$$$`. @@ -217,7 +217,7 @@ Examples: ### replace_pattern -`replace_pattern(target, regex, replacement, replacementPrefix, function)` +`replace_pattern(target, regex, replacement, replacementFormat, function)` The `replace_pattern` function allows replacing all string sections that match a regex pattern with a new value. @@ -225,7 +225,7 @@ The `replace_pattern` function allows replacing all string sections that match a If one or more sections of `target` match `regex` they will get replaced with `replacement`. -The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). `replacementPrefix` is an optional string argument to prefix a `replacement` string with. +The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). `replacementFormat` is an optional string argument that specifies the format of the replacement. The `function` is an optional argument that can take in any Converter that accepts a (`replacement`) string and returns a string. An example is a hash function that replaces a matching regex pattern with the hash value of `replacement`. @@ -237,7 +237,7 @@ Examples: - `replace_pattern(resource.attributes["process.command_line"], "password\\=[^\\s]*(\\s?)", "password=***")` - `replace_pattern(name, "^kube_([0-9A-Za-z]+_)", "k8s.$$1.")` -- `replace_pattern(name, "^kube_([0-9A-Za-z]+_)", "$$1.", "k8s.", SHA256)` +- `replace_pattern(name, "^kube_([0-9A-Za-z]+_)", "$$1.", "k8s.%s", SHA256)` Note that when using OTTL within the collector's configuration file, `$` must be escaped to `$$` to bypass environment variable substitution logic. To input a literal `$` from the configuration file, use `$$$`. diff --git a/pkg/ottl/ottlfuncs/func_replace_all_patterns.go b/pkg/ottl/ottlfuncs/func_replace_all_patterns.go index 175be5e0b197..cc1fc719318d 100644 --- a/pkg/ottl/ottlfuncs/func_replace_all_patterns.go +++ b/pkg/ottl/ottlfuncs/func_replace_all_patterns.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "regexp" + "strings" "go.opentelemetry.io/collector/pdata/pcommon" @@ -23,7 +24,7 @@ type ReplaceAllPatternsArguments[K any] struct { Mode string RegexPattern string Replacement ottl.StringGetter[K] - ReplacementPrefix ottl.Optional[string] + ReplacementFormat ottl.Optional[string] Function ottl.Optional[ottl.FunctionGetter[K]] } @@ -42,10 +43,10 @@ func createReplaceAllPatternsFunction[K any](_ ottl.FunctionContext, oArgs ottl. return nil, fmt.Errorf("ReplaceAllPatternsFactory args must be of type *ReplaceAllPatternsArguments[K]") } - return replaceAllPatterns(args.Target, args.Mode, args.RegexPattern, args.Replacement, args.ReplacementPrefix, args.Function) + return replaceAllPatterns(args.Target, args.Mode, args.RegexPattern, args.Replacement, args.ReplacementFormat, args.Function) } -func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPattern string, replacement ottl.StringGetter[K], replacementPrefix ottl.Optional[string], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) { +func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPattern string, replacement ottl.StringGetter[K], replacementFormat ottl.Optional[string], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) { compiledPattern, err := regexp.Compile(regexPattern) if err != nil { return nil, fmt.Errorf("the regex pattern supplied to replace_all_patterns is not a valid pattern: %w", err) @@ -87,8 +88,12 @@ func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPatt switch mode { case modeValue: if compiledPattern.MatchString(originalValue.Str()) { - if !replacementPrefix.IsEmpty() { // If replacementPrefix is not empty, add it to the replacement value - replacementVal = replacementPrefix.Get() + replacementVal + if !replacementFormat.IsEmpty() { // If replacementFormat is not empty, add it to the replacement value + formatString := replacementFormat.Get() + if !strings.Contains(formatString, "%s") { + return false + } + replacementVal = fmt.Sprintf(replacementFormat.Get(), replacementVal) } updatedString := compiledPattern.ReplaceAllString(originalValue.Str(), replacementVal) updated.PutStr(key, updatedString) @@ -97,8 +102,12 @@ func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPatt } case modeKey: if compiledPattern.MatchString(key) { - if !replacementPrefix.IsEmpty() { // If replacementPrefix is not empty, add it to the replacement value - replacementVal = replacementPrefix.Get() + replacementVal + if !replacementFormat.IsEmpty() { // If replacementFormat is not empty, add it to the replacement value + formatString := replacementFormat.Get() + if !strings.Contains(formatString, "%s") { + return false + } + replacementVal = fmt.Sprintf(replacementFormat.Get(), replacementVal) } updatedKey := compiledPattern.ReplaceAllString(key, replacementVal) originalValue.CopyTo(updated.PutEmpty(updatedKey)) diff --git a/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go b/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go index 7ea56dc8d5e3..f1b517588aca 100644 --- a/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go @@ -44,7 +44,7 @@ func Test_replaceAllPatterns(t *testing.T) { mode string pattern string replacement ottl.StringGetter[pcommon.Map] - replacementPrefix ottl.Optional[string] + replacementFormat ottl.Optional[string] function ottl.Optional[ottl.FunctionGetter[pcommon.Map]] want func(pcommon.Map) }{ @@ -58,7 +58,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "hello {universe}", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: optionalArg, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "4804d6b7f03268e33f78c484977f3d81771220df07cc6aac4ad4868102141fad world") @@ -69,6 +69,44 @@ func Test_replaceAllPatterns(t *testing.T) { expectedMap.PutBool("test6", true) }, }, + { + name: "replace only matches (with replacement format)", + target: target, + mode: modeValue, + pattern: "hello", + replacement: ottl.StandardStringGetter[pcommon.Map]{ + Getter: func(context.Context, pcommon.Map) (interface{}, error) { + return "hello {universe}", nil + }, + }, + replacementFormat: ottl.NewTestingOptional[string]("passwd=%s"), + function: optionalArg, + want: func(expectedMap pcommon.Map) { + expectedMap.PutStr("test", "passwd=4804d6b7f03268e33f78c484977f3d81771220df07cc6aac4ad4868102141fad world") + expectedMap.PutStr("test2", "passwd=passwd=4804d6b7f03268e33f78c484977f3d81771220df07cc6aac4ad4868102141fad") // replacement format is applied twice + expectedMap.PutStr("test3", "goodbye world1 and world2") + expectedMap.PutInt("test4", 1234) + expectedMap.PutDouble("test5", 1234) + expectedMap.PutBool("test6", true) + }, + }, + { + name: "replace only matches (with invalid replacement format)", + target: target, + mode: modeValue, + pattern: "hello", + replacement: ottl.StandardStringGetter[pcommon.Map]{ + Getter: func(context.Context, pcommon.Map) (interface{}, error) { + return "hello {universe}", nil + }, + }, + replacementFormat: ottl.NewTestingOptional[string]("passwd="), + function: optionalArg, + want: func(expectedMap pcommon.Map) { + expectedMap.PutEmpty("test") + expectedMap.Remove("test") + }, + }, { name: "replace only matches", target: target, @@ -79,7 +117,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "hello {universe}", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello {universe} world") @@ -100,7 +138,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "nothing {matches}", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello world") @@ -121,7 +159,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "**** ", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello **** ") @@ -142,7 +180,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "foo", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() @@ -164,7 +202,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "nothing {matches}", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() @@ -186,7 +224,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "test.", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() @@ -208,7 +246,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "world-$1", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() @@ -230,7 +268,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "test-$1", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello world") @@ -251,7 +289,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "$$world-$1", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() @@ -269,7 +307,7 @@ func Test_replaceAllPatterns(t *testing.T) { scenarioMap := pcommon.NewMap() input.CopyTo(scenarioMap) - exprFunc, err := replaceAllPatterns[pcommon.Map](tt.target, tt.mode, tt.pattern, tt.replacement, tt.replacementPrefix, tt.function) + exprFunc, err := replaceAllPatterns[pcommon.Map](tt.target, tt.mode, tt.pattern, tt.replacement, tt.replacementFormat, tt.function) assert.NoError(t, err) _, err = exprFunc(nil, scenarioMap) @@ -295,10 +333,10 @@ func Test_replaceAllPatterns_bad_input(t *testing.T) { return "{replacement}", nil }, } - replacementPrefix := ottl.Optional[string]{} + replacementFormat := ottl.Optional[string]{} function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} - exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, "regexpattern", replacement, replacementPrefix, function) + exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, "regexpattern", replacement, replacementFormat, function) assert.Nil(t, err) _, err = exprFunc(nil, input) @@ -317,10 +355,10 @@ func Test_replaceAllPatterns_bad_function_input(t *testing.T) { return nil, nil }, } - replacementPrefix := ottl.Optional[string]{} + replacementFormat := ottl.Optional[string]{} function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} - exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, "regexp", replacement, replacementPrefix, function) + exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, "regexp", replacement, replacementFormat, function) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -347,10 +385,10 @@ func Test_replaceAllPatterns_bad_function_result(t *testing.T) { }, Fact: StandardConverters[interface{}]()["IsString"], } - replacementPrefix := ottl.Optional[string]{} + replacementFormat := ottl.Optional[string]{} function := ottl.NewTestingOptional[ottl.FunctionGetter[interface{}]](ottlValue) - exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, "regexp", replacement, replacementPrefix, function) + exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, "regexp", replacement, replacementFormat, function) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -369,10 +407,10 @@ func Test_replaceAllPatterns_get_nil(t *testing.T) { return "{anything}", nil }, } - replacementPrefix := ottl.Optional[string]{} + replacementFormat := ottl.Optional[string]{} function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} - exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, "regexp", replacement, replacementPrefix, function) + exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, "regexp", replacement, replacementFormat, function) assert.NoError(t, err) _, err = exprFunc(nil, nil) @@ -391,11 +429,11 @@ func Test_replaceAllPatterns_invalid_pattern(t *testing.T) { return "{anything}", nil }, } - replacementPrefix := ottl.Optional[string]{} + replacementFormat := ottl.Optional[string]{} function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} invalidRegexPattern := "*" - exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, invalidRegexPattern, replacement, replacementPrefix, function) + exprFunc, err := replaceAllPatterns[interface{}](target, modeValue, invalidRegexPattern, replacement, replacementFormat, function) require.Error(t, err) assert.ErrorContains(t, err, "error parsing regexp:") assert.Nil(t, exprFunc) @@ -413,11 +451,11 @@ func Test_replaceAllPatterns_invalid_model(t *testing.T) { return "{anything}", nil }, } - replacementPrefix := ottl.Optional[string]{} + replacementFormat := ottl.Optional[string]{} function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} invalidMode := "invalid" - exprFunc, err := replaceAllPatterns[interface{}](target, invalidMode, "regex", replacement, replacementPrefix, function) + exprFunc, err := replaceAllPatterns[interface{}](target, invalidMode, "regex", replacement, replacementFormat, function) assert.Nil(t, exprFunc) assert.Contains(t, err.Error(), "invalid mode") } diff --git a/pkg/ottl/ottlfuncs/func_replace_pattern.go b/pkg/ottl/ottlfuncs/func_replace_pattern.go index 48ad1007269a..5146a7cdefc7 100644 --- a/pkg/ottl/ottlfuncs/func_replace_pattern.go +++ b/pkg/ottl/ottlfuncs/func_replace_pattern.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "regexp" + "strings" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" ) @@ -15,7 +16,7 @@ type ReplacePatternArguments[K any] struct { Target ottl.GetSetter[K] RegexPattern string Replacement ottl.StringGetter[K] - ReplacementPrefix ottl.Optional[string] // ReplacementPrefix is an optional prefix to add to the replacement value + ReplacementFormat ottl.Optional[string] // ReplacementPrefix is an optional prefix to add to the replacement value Function ottl.Optional[ottl.FunctionGetter[K]] } @@ -34,10 +35,10 @@ func createReplacePatternFunction[K any](_ ottl.FunctionContext, oArgs ottl.Argu return nil, fmt.Errorf("ReplacePatternFactory args must be of type *ReplacePatternArguments[K]") } - return replacePattern(args.Target, args.RegexPattern, args.Replacement, args.ReplacementPrefix, args.Function) + return replacePattern(args.Target, args.RegexPattern, args.Replacement, args.ReplacementFormat, args.Function) } -func replacePattern[K any](target ottl.GetSetter[K], regexPattern string, replacement ottl.StringGetter[K], replacementPrefix ottl.Optional[string], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) { +func replacePattern[K any](target ottl.GetSetter[K], regexPattern string, replacement ottl.StringGetter[K], replacementFormat ottl.Optional[string], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) { compiledPattern, err := regexp.Compile(regexPattern) if err != nil { return nil, fmt.Errorf("the regex pattern supplied to replace_pattern is not a valid pattern: %w", err) @@ -74,8 +75,12 @@ func replacePattern[K any](target ottl.GetSetter[K], regexPattern string, replac } if originalValStr, ok := originalVal.(string); ok { if compiledPattern.MatchString(originalValStr) { - if !replacementPrefix.IsEmpty() { // If replacementPrefix is not empty, add it to the replacement value - replacementVal = replacementPrefix.Get() + replacementVal + if !replacementFormat.IsEmpty() { // If replacementFormat is not empty, add it to the replacement value + formatString := replacementFormat.Get() + if !strings.Contains(formatString, "%s") { + return nil, fmt.Errorf("replacementFormat must be format string with %%s") + } + replacementVal = fmt.Sprintf(replacementFormat.Get(), replacementVal) } updatedStr := compiledPattern.ReplaceAllString(originalValStr, replacementVal) err = target.Set(ctx, tCtx, updatedStr) diff --git a/pkg/ottl/ottlfuncs/func_replace_pattern_test.go b/pkg/ottl/ottlfuncs/func_replace_pattern_test.go index 54ad73798be5..b0dd1d9e9108 100644 --- a/pkg/ottl/ottlfuncs/func_replace_pattern_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_pattern_test.go @@ -39,7 +39,7 @@ func Test_replacePattern(t *testing.T) { target ottl.GetSetter[pcommon.Value] pattern string replacement ottl.StringGetter[pcommon.Value] - replacementPrefix ottl.Optional[string] + replacementFormat ottl.Optional[string] function ottl.Optional[ottl.FunctionGetter[pcommon.Value]] want func(pcommon.Value) }{ @@ -52,12 +52,27 @@ func Test_replacePattern(t *testing.T) { return "passwd=*** ", nil }, }, - replacementPrefix: ottl.NewTestingOptional[string]("passwd="), + replacementFormat: ottl.NewTestingOptional[string]("passwd=%s"), function: optionalArg, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=0f2407f2d83337b1f757eb1754a7643ce0e8fba620bc605c54566cd6dfd838be otherarg=notsensitive key1 key2") }, }, + { + name: "replace regex match (with hash function)", + target: target, + pattern: `passwd\=[^\s]*`, + replacement: ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(context.Context, pcommon.Value) (interface{}, error) { + return "passwd=*** ", nil + }, + }, + replacementFormat: ottl.NewTestingOptional[string]("%s (passwd)"), + function: optionalArg, + want: func(expectedValue pcommon.Value) { + expectedValue.SetStr("application 0f2407f2d83337b1f757eb1754a7643ce0e8fba620bc605c54566cd6dfd838be (passwd) otherarg=notsensitive key1 key2") + }, + }, { name: "replace regex match", target: target, @@ -67,7 +82,7 @@ func Test_replacePattern(t *testing.T) { return "passwd=*** ", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=*** otherarg=notsensitive key1 key2") @@ -82,7 +97,7 @@ func Test_replacePattern(t *testing.T) { return "shouldnotbeinoutput", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=sensitivedtata otherarg=notsensitive key1 key2") @@ -97,7 +112,7 @@ func Test_replacePattern(t *testing.T) { return "**** ", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=sensitivedtata otherarg=notsensitive **** **** ") @@ -112,7 +127,7 @@ func Test_replacePattern(t *testing.T) { return "$1:$2", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd:sensitivedtata otherarg:notsensitive key1 key2") @@ -127,7 +142,7 @@ func Test_replacePattern(t *testing.T) { return "passwd=$$$$$$ ", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=$$$ otherarg=notsensitive key1 key2") @@ -137,7 +152,7 @@ func Test_replacePattern(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { scenarioValue := pcommon.NewValueStr(input.Str()) - exprFunc, err := replacePattern(tt.target, tt.pattern, tt.replacement, tt.replacementPrefix, tt.function) + exprFunc, err := replacePattern(tt.target, tt.pattern, tt.replacement, tt.replacementFormat, tt.function) assert.NoError(t, err) result, err := exprFunc(nil, scenarioValue) @@ -148,6 +163,7 @@ func Test_replacePattern(t *testing.T) { tt.want(expected) assert.Equal(t, expected, scenarioValue) + }) } } @@ -168,10 +184,10 @@ func Test_replacePattern_bad_input(t *testing.T) { return "{replacement}", nil }, } - replacementPrefix := ottl.Optional[string]{} + replacementFormat := ottl.Optional[string]{} function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} - exprFunc, err := replacePattern[interface{}](target, "regexp", replacement, replacementPrefix, function) + exprFunc, err := replacePattern[interface{}](target, "regexp", replacement, replacementFormat, function) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -196,10 +212,10 @@ func Test_replacePattern_bad_function_input(t *testing.T) { return nil, nil }, } - replacementPrefix := ottl.Optional[string]{} + replacementFormat := ottl.Optional[string]{} function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} - exprFunc, err := replacePattern[interface{}](target, "regexp", replacement, replacementPrefix, function) + exprFunc, err := replacePattern[interface{}](target, "regexp", replacement, replacementFormat, function) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -230,10 +246,10 @@ func Test_replacePattern_bad_function_result(t *testing.T) { }, Fact: StandardConverters[interface{}]()["IsString"], } - replacementPrefix := ottl.Optional[string]{} + replacementFormat := ottl.Optional[string]{} function := ottl.NewTestingOptional[ottl.FunctionGetter[interface{}]](ottlValue) - exprFunc, err := replacePattern[interface{}](target, "regexp", replacement, replacementPrefix, function) + exprFunc, err := replacePattern[interface{}](target, "regexp", replacement, replacementFormat, function) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -257,10 +273,10 @@ func Test_replacePattern_get_nil(t *testing.T) { return "{anything}", nil }, } - replacementPrefix := ottl.Optional[string]{} + replacementFormat := ottl.Optional[string]{} function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} - exprFunc, err := replacePattern[interface{}](target, `nomatch\=[^\s]*(\s?)`, replacement, replacementPrefix, function) + exprFunc, err := replacePattern[interface{}](target, `nomatch\=[^\s]*(\s?)`, replacement, replacementFormat, function) assert.NoError(t, err) result, err := exprFunc(nil, nil) @@ -284,11 +300,44 @@ func Test_replacePatterns_invalid_pattern(t *testing.T) { return "{anything}", nil }, } - replacementPrefix := ottl.Optional[string]{} + replacementFormat := ottl.Optional[string]{} function := ottl.Optional[ottl.FunctionGetter[interface{}]]{} invalidRegexPattern := "*" - _, err := replacePattern[interface{}](target, invalidRegexPattern, replacement, replacementPrefix, function) + _, err := replacePattern[interface{}](target, invalidRegexPattern, replacement, replacementFormat, function) require.Error(t, err) assert.ErrorContains(t, err, "error parsing regexp:") } + +func Test_replacePattern_bad_format_string(t *testing.T) { + input := pcommon.NewValueStr("application passwd=sensitivedtata otherarg=notsensitive key1 key2") + target := &ottl.StandardGetSetter[pcommon.Value]{ + Getter: func(ctx context.Context, tCtx pcommon.Value) (interface{}, error) { + return tCtx.Str(), nil + }, + Setter: func(ctx context.Context, tCtx pcommon.Value, val interface{}) error { + tCtx.SetStr(val.(string)) + return nil + }, + } + replacement := &ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(context.Context, pcommon.Value) (interface{}, error) { + return "passwd=*** ", nil + }, + } + ottlValue := ottl.StandardFunctionGetter[pcommon.Value]{ + FCtx: ottl.FunctionContext{ + Set: componenttest.NewNopTelemetrySettings(), + }, + Fact: StandardConverters[pcommon.Value]()["SHA256"], + } + replacementFormat := ottl.NewTestingOptional[string]("passwd=") // This is not a valid format string + function := ottl.NewTestingOptional[ottl.FunctionGetter[pcommon.Value]](ottlValue) + + exprFunc, err := replacePattern[pcommon.Value](target, `passwd\=[^\s]*`, replacement, replacementFormat, function) + assert.NoError(t, err) + result, err := exprFunc(nil, input) + require.Error(t, err) + assert.ErrorContains(t, err, "replacementFormat must be format string with %s") + assert.Nil(t, result) +}