Skip to content

Commit

Permalink
Include an option argument to revert to old hashing behavior
Browse files Browse the repository at this point in the history
  • Loading branch information
@rnishtala-sumo
rnishtala-sumo committed Jan 23, 2024
1 parent 60f2970 commit 8600fc1
Show file tree
Hide file tree
Showing 5 changed files with 152 additions and 91 deletions.
4 changes: 2 additions & 2 deletions pkg/ottl/ottlfuncs/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,7 @@ The `replace_all_patterns` function replaces any segments in a string value or k

If one or more sections of `target` match `regex` they will get replaced with `replacement`.

The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand).
The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). It can contain upto 9 capture groups ($1-$9).

The `function` is an optional argument that can take in any Converter that accepts a (`replacement`) string and returns a string. An example is a hash function that replaces any matching regex pattern with the hash value of `replacement`.

Expand Down Expand Up @@ -323,7 +323,7 @@ The `replace_pattern` function allows replacing all string sections that match a

If one or more sections of `target` match `regex` they will get replaced with `replacement`.

The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand).
The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). It can contain upto 9 capture groups ($1-$9).

The `function` is an optional argument that can take in any Converter that accepts a (`replacement`) string and returns a string. An example is a hash function that replaces a matching regex pattern with the hash value of `replacement`.

Expand Down
19 changes: 10 additions & 9 deletions pkg/ottl/ottlfuncs/func_replace_all_patterns.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,12 @@ const (
)

type ReplaceAllPatternsArguments[K any] struct {
Target ottl.PMapGetter[K]
Mode string
RegexPattern string
Replacement ottl.StringGetter[K]
Function ottl.Optional[ottl.FunctionGetter[K]]
Target ottl.PMapGetter[K]
Mode string
RegexPattern string
Replacement ottl.StringGetter[K]
Function ottl.Optional[ottl.FunctionGetter[K]]
IncludeNonCapture ottl.Optional[bool]
}

func NewReplaceAllPatternsFactory[K any]() ottl.Factory[K] {
Expand All @@ -37,10 +38,10 @@ func createReplaceAllPatternsFunction[K any](_ ottl.FunctionContext, oArgs ottl.
return nil, fmt.Errorf("ReplaceAllPatternsFactory args must be of type *ReplaceAllPatternsArguments[K]")
}

return replaceAllPatterns(args.Target, args.Mode, args.RegexPattern, args.Replacement, args.Function)
return replaceAllPatterns(args.Target, args.Mode, args.RegexPattern, args.Replacement, args.Function, args.IncludeNonCapture)
}

func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) {
func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]], includeNonCapture ottl.Optional[bool]) (ottl.ExprFunc[K], error) {
compiledPattern, err := regexp.Compile(regexPattern)
if err != nil {
return nil, fmt.Errorf("the regex pattern supplied to replace_all_patterns is not a valid pattern: %w", err)
Expand All @@ -66,7 +67,7 @@ func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPatt
case modeValue:
if compiledPattern.MatchString(originalValue.Str()) {
if !fn.IsEmpty() {
updatedString, err := applyOptReplaceFunction(ctx, tCtx, compiledPattern, fn, originalValue.Str(), replacementVal)
updatedString, err := applyOptReplaceFunction(ctx, tCtx, compiledPattern, fn, originalValue.Str(), replacementVal, includeNonCapture)
if err != nil {
return false
}
Expand All @@ -81,7 +82,7 @@ func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPatt
case modeKey:
if compiledPattern.MatchString(key) {
if !fn.IsEmpty() {
updatedString, err := applyOptReplaceFunction(ctx, tCtx, compiledPattern, fn, key, replacementVal)
updatedString, err := applyOptReplaceFunction(ctx, tCtx, compiledPattern, fn, key, replacementVal, includeNonCapture)
if err != nil {
return false
}
Expand Down
125 changes: 83 additions & 42 deletions pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,14 @@ func Test_replaceAllPatterns(t *testing.T) {
}

tests := []struct {
name string
target ottl.PMapGetter[pcommon.Map]
mode string
pattern string
replacement ottl.StringGetter[pcommon.Map]
function ottl.Optional[ottl.FunctionGetter[pcommon.Map]]
want func(pcommon.Map)
name string
target ottl.PMapGetter[pcommon.Map]
mode string
pattern string
replacement ottl.StringGetter[pcommon.Map]
function ottl.Optional[ottl.FunctionGetter[pcommon.Map]]
includeNonCapture ottl.Optional[bool]
want func(pcommon.Map)
}{
{
name: "replace only matches (with hash function)",
Expand All @@ -57,7 +58,8 @@ func Test_replaceAllPatterns(t *testing.T) {
return "hello {universe}", nil
},
},
function: optionalArg,
function: optionalArg,
includeNonCapture: ottl.Optional[bool]{},
want: func(expectedMap pcommon.Map) {
expectedMap.PutStr("test", "hash(hello {universe}) world")
expectedMap.PutStr("test2", "hash(hello {universe})")
Expand All @@ -77,7 +79,8 @@ func Test_replaceAllPatterns(t *testing.T) {
return "$1", nil
},
},
function: optionalArg,
function: optionalArg,
includeNonCapture: ottl.Optional[bool]{},
want: func(expectedMap pcommon.Map) {
expectedMap.PutStr("test", "hash(hello) world")
expectedMap.PutStr("test2", "hash(hello)")
Expand All @@ -97,7 +100,8 @@ func Test_replaceAllPatterns(t *testing.T) {
return "$1", nil
},
},
function: optionalArg,
function: optionalArg,
includeNonCapture: ottl.Optional[bool]{},
want: func(expectedMap pcommon.Map) {
expectedMap.PutStr("test", "hash() world")
expectedMap.PutStr("test2", "hash()")
Expand All @@ -117,7 +121,8 @@ func Test_replaceAllPatterns(t *testing.T) {
return "$1", nil
},
},
function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{},
function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{},
includeNonCapture: ottl.Optional[bool]{},
want: func(expectedMap pcommon.Map) {
expectedMap.PutStr("test", " world")
expectedMap.PutStr("test2", "")
Expand Down Expand Up @@ -157,7 +162,8 @@ func Test_replaceAllPatterns(t *testing.T) {
return "nothing {matches}", nil
},
},
function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{},
function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{},
includeNonCapture: ottl.Optional[bool]{},
want: func(expectedMap pcommon.Map) {
expectedMap.PutStr("test", "hello world")
expectedMap.PutStr("test2", "hello")
Expand All @@ -177,7 +183,8 @@ func Test_replaceAllPatterns(t *testing.T) {
return "**** ", nil
},
},
function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{},
function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{},
includeNonCapture: ottl.Optional[bool]{},
want: func(expectedMap pcommon.Map) {
expectedMap.PutStr("test", "hello **** ")
expectedMap.PutStr("test2", "hello")
Expand All @@ -197,7 +204,8 @@ func Test_replaceAllPatterns(t *testing.T) {
return "blue-$1 and blue-$2", nil
},
},
function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{},
function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{},
includeNonCapture: ottl.Optional[bool]{},
want: func(expectedMap pcommon.Map) {
expectedMap.PutStr("test", "hello world")
expectedMap.PutStr("test2", "hello")
Expand All @@ -217,7 +225,8 @@ func Test_replaceAllPatterns(t *testing.T) {
return "blue-$1 and blue-$2", nil
},
},
function: optionalArg,
function: optionalArg,
includeNonCapture: ottl.Optional[bool]{},
want: func(expectedMap pcommon.Map) {
expectedMap.PutStr("test", "hello world")
expectedMap.PutStr("test2", "hello")
Expand All @@ -227,6 +236,27 @@ func Test_replaceAllPatterns(t *testing.T) {
expectedMap.PutBool("test6", true)
},
},
{
name: "regex match old behavior(with multiple capture groups)",
target: target,
mode: modeValue,
pattern: `(world1) and (world2)`,
replacement: ottl.StandardStringGetter[pcommon.Map]{
Getter: func(context.Context, pcommon.Map) (any, error) {
return "blue-$1 and blue-$2", nil
},
},
function: optionalArg,
includeNonCapture: ottl.NewTestingOptional[bool](true),
want: func(expectedMap pcommon.Map) {
expectedMap.PutStr("test", "hello world")
expectedMap.PutStr("test2", "hello")
expectedMap.PutStr("test3", "goodbye hash(blue-world1 and blue-world2)")
expectedMap.PutInt("test4", 1234)
expectedMap.PutDouble("test5", 1234)
expectedMap.PutBool("test6", true)
},
},
{
name: "regex multiple matches (with multiple capture groups)",
target: target,
Expand All @@ -237,7 +267,8 @@ func Test_replaceAllPatterns(t *testing.T) {
return "blue-$1 green-$1 and blue-$2 green-$2", nil
},
},
function: optionalArg,
function: optionalArg,
includeNonCapture: ottl.Optional[bool]{},
want: func(expectedMap pcommon.Map) {
expectedMap.PutStr("test", "hello world")
expectedMap.PutStr("test2", "hello")
Expand All @@ -257,7 +288,8 @@ func Test_replaceAllPatterns(t *testing.T) {
return "blue-$1", nil
},
},
function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{},
function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{},
includeNonCapture: ottl.Optional[bool]{},
want: func(expectedMap pcommon.Map) {
expectedMap.PutStr("test", "hello world")
expectedMap.PutStr("test2", "hello")
Expand All @@ -277,7 +309,8 @@ func Test_replaceAllPatterns(t *testing.T) {
return "blue-$1", nil
},
},
function: optionalArg,
function: optionalArg,
includeNonCapture: ottl.Optional[bool]{},
want: func(expectedMap pcommon.Map) {
expectedMap.PutStr("test", "hello world")
expectedMap.PutStr("test2", "hello")
Expand All @@ -297,7 +330,8 @@ func Test_replaceAllPatterns(t *testing.T) {
return "$1", nil
},
},
function: optionalArg,
function: optionalArg,
includeNonCapture: ottl.Optional[bool]{},
want: func(expectedMap pcommon.Map) {
expectedMap.PutStr("test", "hello world")
expectedMap.PutStr("test2", "hello")
Expand All @@ -317,7 +351,8 @@ func Test_replaceAllPatterns(t *testing.T) {
return "$2", nil
},
},
function: optionalArg,
function: optionalArg,
includeNonCapture: ottl.Optional[bool]{},
want: func(expectedMap pcommon.Map) {
expectedMap.PutStr("test", "hello world")
expectedMap.PutStr("test2", "hello")
Expand All @@ -337,7 +372,8 @@ func Test_replaceAllPatterns(t *testing.T) {
return "$1", nil
},
},
function: optionalArg,
function: optionalArg,
includeNonCapture: ottl.Optional[bool]{},
want: func(expectedMap pcommon.Map) {
expectedMap.PutStr("test", "hello world")
expectedMap.PutStr("test2", "hello")
Expand All @@ -357,7 +393,8 @@ func Test_replaceAllPatterns(t *testing.T) {
return "foo", nil
},
},
function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{},
function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{},
includeNonCapture: ottl.Optional[bool]{},
want: func(expectedMap pcommon.Map) {
expectedMap.Clear()
expectedMap.PutStr("test", "hello world")
Expand All @@ -378,7 +415,8 @@ func Test_replaceAllPatterns(t *testing.T) {
return "nothing {matches}", nil
},
},
function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{},
function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{},
includeNonCapture: ottl.Optional[bool]{},
want: func(expectedMap pcommon.Map) {
expectedMap.Clear()
expectedMap.PutStr("test", "hello world")
Expand All @@ -399,7 +437,8 @@ func Test_replaceAllPatterns(t *testing.T) {
return "test.", nil
},
},
function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{},
function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{},
includeNonCapture: ottl.Optional[bool]{},
want: func(expectedMap pcommon.Map) {
expectedMap.Clear()
expectedMap.PutStr("test.", "hello world")
Expand All @@ -420,7 +459,8 @@ func Test_replaceAllPatterns(t *testing.T) {
return "world-$1", nil
},
},
function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{},
function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{},
includeNonCapture: ottl.Optional[bool]{},
want: func(expectedMap pcommon.Map) {
expectedMap.Clear()
expectedMap.PutStr("test", "hello world")
Expand All @@ -441,7 +481,8 @@ func Test_replaceAllPatterns(t *testing.T) {
return "test-$1", nil
},
},
function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{},
function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{},
includeNonCapture: ottl.Optional[bool]{},
want: func(expectedMap pcommon.Map) {
expectedMap.PutStr("test", "hello world")
expectedMap.PutStr("test-2", "hello")
Expand All @@ -461,7 +502,8 @@ func Test_replaceAllPatterns(t *testing.T) {
return "$$world-$1", nil
},
},
function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{},
function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{},
includeNonCapture: ottl.Optional[bool]{},
want: func(expectedMap pcommon.Map) {
expectedMap.Clear()
expectedMap.PutStr("test", "hello world")
Expand All @@ -478,7 +520,7 @@ func Test_replaceAllPatterns(t *testing.T) {
scenarioMap := pcommon.NewMap()
input.CopyTo(scenarioMap)

exprFunc, err := replaceAllPatterns[pcommon.Map](tt.target, tt.mode, tt.pattern, tt.replacement, tt.function)
exprFunc, err := replaceAllPatterns[pcommon.Map](tt.target, tt.mode, tt.pattern, tt.replacement, tt.function, tt.includeNonCapture)
assert.NoError(t, err)

_, err = exprFunc(nil, scenarioMap)
Expand All @@ -505,10 +547,9 @@ func Test_replaceAllPatterns_bad_input(t *testing.T) {
},
}
function := ottl.Optional[ottl.FunctionGetter[any]]{}

exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexpattern", replacement, function)
assert.NoError(t, err)

includeNonCapture := ottl.Optional[bool]{}
exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexpattern", replacement, function, includeNonCapture)
assert.Nil(t, err)
_, err = exprFunc(nil, input)
assert.Error(t, err)
}
Expand All @@ -526,8 +567,8 @@ func Test_replaceAllPatterns_bad_function_input(t *testing.T) {
},
}
function := ottl.Optional[ottl.FunctionGetter[any]]{}

exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function)
includeNonCapture := ottl.Optional[bool]{}
exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function, includeNonCapture)
assert.NoError(t, err)

result, err := exprFunc(nil, input)
Expand Down Expand Up @@ -555,8 +596,8 @@ func Test_replaceAllPatterns_bad_function_result(t *testing.T) {
Fact: StandardConverters[any]()["IsString"],
}
function := ottl.NewTestingOptional[ottl.FunctionGetter[any]](ottlValue)

exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function)
includeNonCapture := ottl.Optional[bool]{}
exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function, includeNonCapture)
assert.NoError(t, err)

result, err := exprFunc(nil, input)
Expand All @@ -576,8 +617,8 @@ func Test_replaceAllPatterns_get_nil(t *testing.T) {
},
}
function := ottl.Optional[ottl.FunctionGetter[any]]{}

exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function)
includeNonCapture := ottl.Optional[bool]{}
exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function, includeNonCapture)
assert.NoError(t, err)

_, err = exprFunc(nil, nil)
Expand All @@ -597,9 +638,9 @@ func Test_replaceAllPatterns_invalid_pattern(t *testing.T) {
},
}
function := ottl.Optional[ottl.FunctionGetter[any]]{}

includeNonCapture := ottl.Optional[bool]{}
invalidRegexPattern := "*"
exprFunc, err := replaceAllPatterns[any](target, modeValue, invalidRegexPattern, replacement, function)
exprFunc, err := replaceAllPatterns[any](target, modeValue, invalidRegexPattern, replacement, function, includeNonCapture)
require.Error(t, err)
assert.ErrorContains(t, err, "error parsing regexp:")
assert.Nil(t, exprFunc)
Expand All @@ -618,9 +659,9 @@ func Test_replaceAllPatterns_invalid_model(t *testing.T) {
},
}
function := ottl.Optional[ottl.FunctionGetter[any]]{}

includeNonCapture := ottl.Optional[bool]{}
invalidMode := "invalid"
exprFunc, err := replaceAllPatterns[any](target, invalidMode, "regex", replacement, function)
exprFunc, err := replaceAllPatterns[any](target, invalidMode, "regex", replacement, function, includeNonCapture)
assert.Nil(t, exprFunc)
assert.Contains(t, err.Error(), "invalid mode")
}
Loading

0 comments on commit 8600fc1

Please sign in to comment.