From 97467be5623747942e847d9b6fbc0a867ed795e0 Mon Sep 17 00:00:00 2001 From: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> Date: Tue, 22 Nov 2022 16:24:25 -0700 Subject: [PATCH 01/13] Add ParseToMap factory function --- pkg/ottl/ottlfuncs/README.md | 74 +++++--- pkg/ottl/ottlfuncs/func_parse_to_map.go | 103 +++++++++++ pkg/ottl/ottlfuncs/func_parse_to_map_test.go | 183 +++++++++++++++++++ 3 files changed, 336 insertions(+), 24 deletions(-) create mode 100644 pkg/ottl/ottlfuncs/func_parse_to_map.go create mode 100644 pkg/ottl/ottlfuncs/func_parse_to_map_test.go diff --git a/pkg/ottl/ottlfuncs/README.md b/pkg/ottl/ottlfuncs/README.md index 9ce7ac709204..fbf1684a7fa7 100644 --- a/pkg/ottl/ottlfuncs/README.md +++ b/pkg/ottl/ottlfuncs/README.md @@ -4,12 +4,13 @@ The following functions are intended to be used in implementations of the OpenTe Factory Functions - [Concat](#concat) +- [ConvertCase](#convertcase) - [Int](#int) - [IsMatch](#ismatch) +- [ParseToMap](#ParseToMap) - [SpanID](#spanid) - [Split](#split) - [TraceID](#traceid) -- [ConvertCase](#convertcase) Functions - [delete_key](#delete_key) @@ -43,6 +44,29 @@ Examples: - `Concat(["HTTP method is: ", attributes["http.method"]], "")` +## ConvertCase + +`ConvertCase(target, toCase)` + +The `ConvertCase` factory function converts the `target` string into the desired case `toCase`. + +`target` is a string. `toCase` is a string. + +If the `target` is not a string or does not exist, the `ConvertCase` factory function will return `nil`. + +`toCase` can be: + +- `lower`: Converts the `target` string to lowercase (e.g. `MY_METRIC` to `my_metric`) +- `upper`: Converts the `target` string to uppercase (e.g. `my_metric` to `MY_METRIC`) +- `snake`: Converts the `target` string to snakecase (e.g. `myMetric` to `my_metric`) +- `camel`: Converts the `target` string to camelcase (e.g. `my_metric` to `MyMetric`) + +If `toCase` is any value other than the options above, the `ConvertCase` factory function will return an error during collector startup. + +Examples: + +- `ConvertCase(metric.name, "snake")` + ## Int `Int(value)` @@ -85,6 +109,31 @@ Examples: - `IsMatch("string", ".*ring")` +## ParseToMap + +`ParseToMap(target, inputFormat)` + +The `ParseToMap` factory function returns a `pcommon.Map` struct that is a result of parsing the target string using the specified format. + +`target` is a Getter that returns a string. `inputFormat` is a string that specifies what format to use when parsing. Valid values for `inputFormat` are: `json`. + +If `json` is used the target is unmarshalled using [jsoniter](https://github.com/json-iterator/go). Each JSON type is converted into a `pdata.Value` using the following map: + +``` +JSON boolean -> bool +JSON number -> float64 +JSON string -> string +JSON null -> nil +JSON arrays -> pdata.SliceValue +JSON objects -> map[string]any +``` + +Examples: + +- `ParseToMap("{\"attr\":true}", "json")` +- `ParseToMap(attributes["kubernetes"], "json")` +- `ParseToMap(body, "json")` + ## SpanID `SpanID(bytes)` @@ -123,29 +172,6 @@ Examples: - `TraceID(0x00000000000000000000000000000000)` -## ConvertCase - -`ConvertCase(target, toCase)` - -The `ConvertCase` factory function converts the `target` string into the desired case `toCase`. - -`target` is a string. `toCase` is a string. - -If the `target` is not a string or does not exist, the `ConvertCase` factory function will return `nil`. - -`toCase` can be: - -- `lower`: Converts the `target` string to lowercase (e.g. `MY_METRIC` to `my_metric`) -- `upper`: Converts the `target` string to uppercase (e.g. `my_metric` to `MY_METRIC`) -- `snake`: Converts the `target` string to snakecase (e.g. `myMetric` to `my_metric`) -- `camel`: Converts the `target` string to camelcase (e.g. `my_metric` to `MyMetric`) - -If `toCase` is any value other than the options above, the `ConvertCase` factory function will return an error during collector startup. - -Examples: - -- `ConvertCase(metric.name, "snake")` - ## delete_key `delete_key(target, key)` diff --git a/pkg/ottl/ottlfuncs/func_parse_to_map.go b/pkg/ottl/ottlfuncs/func_parse_to_map.go new file mode 100644 index 000000000000..a6dee61c0b66 --- /dev/null +++ b/pkg/ottl/ottlfuncs/func_parse_to_map.go @@ -0,0 +1,103 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ottlfuncs // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/ottlfuncs" + +import ( + "context" + "fmt" + + jsoniter "github.com/json-iterator/go" + "go.opentelemetry.io/collector/pdata/pcommon" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" +) + +const InputFormatJSON = "json" + +func ParseToMap[K any](target ottl.Getter[K], inputFormat string) (ottl.ExprFunc[K], error) { + if inputFormat != InputFormatJSON { + return nil, fmt.Errorf("unknown inputFormat, %v", inputFormat) + } + return func(ctx context.Context, tCtx K) (interface{}, error) { + targetVal, err := target.Get(ctx, tCtx) + if err != nil { + return nil, err + } + if valStr, ok := targetVal.(string); ok { + switch inputFormat { + case InputFormatJSON: + return parseJSON(valStr) + default: + return nil, fmt.Errorf("unknown inputFormat, %v", inputFormat) + } + } + return nil, nil + }, nil +} + +func parseJSON(jsonStr string) (pcommon.Map, error) { + var parsedValue map[string]interface{} + err := jsoniter.UnmarshalFromString(jsonStr, &parsedValue) + if err != nil { + return pcommon.Map{}, err + } + result := pcommon.NewMap() + for k, v := range parsedValue { + attrVal := pcommon.NewValueEmpty() + err = setValue(attrVal, v) + if err != nil { + return pcommon.Map{}, err + } + attrVal.CopyTo(result.PutEmpty(k)) + } + return result, nil +} + +func setValue(value pcommon.Value, val interface{}) error { + switch v := val.(type) { + case string: + value.SetStr(v) + case bool: + value.SetBool(v) + case float64: + value.SetDouble(v) + case nil: + case []interface{}: + emptySlice := value.SetEmptySlice() + err := setSlice(emptySlice, v) + if err != nil { + return err + } + case map[string]interface{}: + err := value.SetEmptyMap().FromRaw(v) + if err != nil { + return err + } + default: + return fmt.Errorf("unknown type, %T", v) + } + return nil +} + +func setSlice(slice pcommon.Slice, value []interface{}) error { + for _, item := range value { + emptyValue := slice.AppendEmpty() + err := setValue(emptyValue, item) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/ottl/ottlfuncs/func_parse_to_map_test.go b/pkg/ottl/ottlfuncs/func_parse_to_map_test.go new file mode 100644 index 000000000000..8ee7f2269a49 --- /dev/null +++ b/pkg/ottl/ottlfuncs/func_parse_to_map_test.go @@ -0,0 +1,183 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ottlfuncs // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/ottlfuncs" + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/pdata/pcommon" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" +) + +func Test_ParseToMap(t *testing.T) { + tests := []struct { + name string + target ottl.Getter[any] + inputFormat string + want func(pcommon.Map) + }{ + { + name: "handle string", + inputFormat: "json", + target: ottl.StandardGetSetter[any]{ + Getter: func(ctx context.Context, tCtx any) (interface{}, error) { + return `{"test":"string value"}`, nil + }, + }, + want: func(expectedMap pcommon.Map) { + expectedMap.PutStr("test", "string value") + }, + }, + { + name: "handle bool", + inputFormat: "json", + target: ottl.StandardGetSetter[any]{ + Getter: func(ctx context.Context, tCtx any) (interface{}, error) { + return `{"test":true}`, nil + }, + }, + want: func(expectedMap pcommon.Map) { + expectedMap.PutBool("test", true) + }, + }, + { + name: "handle int", + inputFormat: "json", + target: ottl.StandardGetSetter[any]{ + Getter: func(ctx context.Context, tCtx any) (interface{}, error) { + return `{"test":1}`, nil + }, + }, + want: func(expectedMap pcommon.Map) { + expectedMap.PutDouble("test", 1) + }, + }, + { + name: "handle float", + inputFormat: "json", + target: ottl.StandardGetSetter[any]{ + Getter: func(ctx context.Context, tCtx any) (interface{}, error) { + return `{"test":1.1}`, nil + }, + }, + want: func(expectedMap pcommon.Map) { + expectedMap.PutDouble("test", 1.1) + }, + }, + { + name: "handle nil", + inputFormat: "json", + target: ottl.StandardGetSetter[any]{ + Getter: func(ctx context.Context, tCtx any) (interface{}, error) { + return `{"test":null}`, nil + }, + }, + want: func(expectedMap pcommon.Map) { + expectedMap.PutEmpty("test") + }, + }, + { + name: "handle array", + inputFormat: "json", + target: ottl.StandardGetSetter[any]{ + Getter: func(ctx context.Context, tCtx any) (interface{}, error) { + return `{"test":["string","value"]}`, nil + }, + }, + want: func(expectedMap pcommon.Map) { + emptySlice := expectedMap.PutEmptySlice("test") + emptySlice.AppendEmpty().SetStr("string") + emptySlice.AppendEmpty().SetStr("value") + }, + }, + { + name: "handle nested object", + inputFormat: "json", + target: ottl.StandardGetSetter[any]{ + Getter: func(ctx context.Context, tCtx any) (interface{}, error) { + return `{"test":{"nested":"true"}}`, nil + }, + }, + want: func(expectedMap pcommon.Map) { + newMap := expectedMap.PutEmptyMap("test") + newMap.PutStr("nested", "true") + }, + }, + { + name: "updates existing", + inputFormat: "json", + target: ottl.StandardGetSetter[any]{ + Getter: func(ctx context.Context, tCtx any) (interface{}, error) { + return `{"existing":"pass"}`, nil + }, + }, + want: func(expectedMap pcommon.Map) { + expectedMap.PutStr("existing", "pass") + }, + }, + { + name: "complex", + inputFormat: "json", + target: ottl.StandardGetSetter[any]{ + Getter: func(ctx context.Context, tCtx any) (interface{}, error) { + return `{"test1":{"nested":"true"},"test2":"string","test3":1,"test4":1.1,"test5":[[1], [2, 3],[]],"test6":null}`, nil + }, + }, + want: func(expectedMap pcommon.Map) { + newMap := expectedMap.PutEmptyMap("test1") + newMap.PutStr("nested", "true") + expectedMap.PutStr("test2", "string") + expectedMap.PutDouble("test3", 1) + expectedMap.PutDouble("test4", 1.1) + slice := expectedMap.PutEmptySlice("test5") + slice0 := slice.AppendEmpty().SetEmptySlice() + slice0.AppendEmpty().SetDouble(1) + slice1 := slice.AppendEmpty().SetEmptySlice() + slice1.AppendEmpty().SetDouble(2) + slice1.AppendEmpty().SetDouble(3) + slice.AppendEmpty().SetEmptySlice() + expectedMap.PutEmpty("test6") + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + exprFunc, err := ParseToMap(tt.target, "json") + assert.NoError(t, err) + + result, err := exprFunc(context.Background(), nil) + assert.NoError(t, err) + + resultMap, ok := result.(pcommon.Map) + if !ok { + assert.Fail(t, "pcommon.Map not returned") + } + + expected := pcommon.NewMap() + tt.want(expected) + + assert.Equal(t, expected.Len(), resultMap.Len()) + expected.Range(func(k string, v pcommon.Value) bool { + ev, _ := expected.Get(k) + av, _ := resultMap.Get(k) + assert.Equal(t, ev, av) + return true + }) + }) + } +} From d1825d4be96f7f329a0e75221eb73d36ea08ac53 Mon Sep 17 00:00:00 2001 From: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> Date: Tue, 22 Nov 2022 16:28:56 -0700 Subject: [PATCH 02/13] add changelog entry --- .chloggen/ottl-parse-to-map.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100755 .chloggen/ottl-parse-to-map.yaml diff --git a/.chloggen/ottl-parse-to-map.yaml b/.chloggen/ottl-parse-to-map.yaml new file mode 100755 index 000000000000..5f9d1a72bb26 --- /dev/null +++ b/.chloggen/ottl-parse-to-map.yaml @@ -0,0 +1,16 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: pkg/ottl + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add new `ParseToMap` function that can convert a json string into `pcommon.Map`. + +# One or more tracking issues related to the change +issues: [16444] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: From c423d6e97c8ccefa49b944a57cdbfddacc0d1369 Mon Sep 17 00:00:00 2001 From: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> Date: Tue, 22 Nov 2022 16:41:06 -0700 Subject: [PATCH 03/13] run go mod tidy --- pkg/ottl/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ottl/go.mod b/pkg/ottl/go.mod index c7f2a9518213..0abc65d90223 100644 --- a/pkg/ottl/go.mod +++ b/pkg/ottl/go.mod @@ -6,6 +6,7 @@ require ( github.com/alecthomas/participle/v2 v2.0.0-beta.5 github.com/gobwas/glob v0.2.3 github.com/iancoleman/strcase v0.2.0 + github.com/json-iterator/go v1.1.12 github.com/stretchr/testify v1.8.1 go.opentelemetry.io/collector/component v0.65.0 go.opentelemetry.io/collector/pdata v0.65.0 @@ -20,7 +21,6 @@ require ( github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/json-iterator/go v1.1.12 // indirect github.com/knadh/koanf v1.4.4 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect From 3dfd149ad2d2d66ad55a49191ae759bb93e0792e Mon Sep 17 00:00:00 2001 From: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> Date: Wed, 23 Nov 2022 10:41:49 -0700 Subject: [PATCH 04/13] Switch to json-specific parser --- pkg/ottl/ottlfuncs/README.md | 23 ++++++---- ...unc_parse_to_map.go => func_parse_json.go} | 46 +++++++------------ ...to_map_test.go => func_parse_json_test.go} | 38 ++++++--------- 3 files changed, 44 insertions(+), 63 deletions(-) rename pkg/ottl/ottlfuncs/{func_parse_to_map.go => func_parse_json.go} (67%) rename pkg/ottl/ottlfuncs/{func_parse_to_map_test.go => func_parse_json_test.go} (86%) diff --git a/pkg/ottl/ottlfuncs/README.md b/pkg/ottl/ottlfuncs/README.md index 60ab294fe73d..60ad12ed802e 100644 --- a/pkg/ottl/ottlfuncs/README.md +++ b/pkg/ottl/ottlfuncs/README.md @@ -37,7 +37,7 @@ List of available Factory Functions: - [ConvertCase](#convertcase) - [Int](#int) - [IsMatch](#ismatch) -- [ParseToMap](#ParseToMap) +- [ParseJSON](#ParseJSON) - [SpanID](#spanid) - [Split](#split) - [TraceID](#traceid) @@ -127,15 +127,16 @@ Examples: - `IsMatch("string", ".*ring")` -### ParseToMap +### ParseJSON -`ParseToMap(target, inputFormat)` +`ParseJSON(target)` -The `ParseToMap` factory function returns a `pcommon.Map` struct that is a result of parsing the target string using the specified format. +The `ParseJSON` factory function returns a `pcommon.Map` struct that is a result of parsing the target string as JSON -`target` is a Getter that returns a string. `inputFormat` is a string that specifies what format to use when parsing. Valid values for `inputFormat` are: `json`. +`target` is a Getter that returns a string. This string should be in json format. -If `json` is used the target is unmarshalled using [jsoniter](https://github.com/json-iterator/go). Each JSON type is converted into a `pdata.Value` using the following map: +Unmarshalling is done using [jsoniter](https://github.com/json-iterator/go). +Each JSON type is converted into a `pdata.Value` using the following map: ``` JSON boolean -> bool @@ -148,9 +149,13 @@ JSON objects -> map[string]any Examples: -- `ParseToMap("{\"attr\":true}", "json")` -- `ParseToMap(attributes["kubernetes"], "json")` -- `ParseToMap(body, "json")` +- `ParseToMap("{\"attr\":true}")` + + +- `ParseToMap(attributes["kubernetes"])` + + +- `ParseToMap(body)` ### SpanID diff --git a/pkg/ottl/ottlfuncs/func_parse_to_map.go b/pkg/ottl/ottlfuncs/func_parse_json.go similarity index 67% rename from pkg/ottl/ottlfuncs/func_parse_to_map.go rename to pkg/ottl/ottlfuncs/func_parse_json.go index a6dee61c0b66..d5325274a47c 100644 --- a/pkg/ottl/ottlfuncs/func_parse_to_map.go +++ b/pkg/ottl/ottlfuncs/func_parse_json.go @@ -24,47 +24,33 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" ) -const InputFormatJSON = "json" - -func ParseToMap[K any](target ottl.Getter[K], inputFormat string) (ottl.ExprFunc[K], error) { - if inputFormat != InputFormatJSON { - return nil, fmt.Errorf("unknown inputFormat, %v", inputFormat) - } +func ParseJSON[K any](target ottl.Getter[K]) (ottl.ExprFunc[K], error) { return func(ctx context.Context, tCtx K) (interface{}, error) { targetVal, err := target.Get(ctx, tCtx) if err != nil { return nil, err } - if valStr, ok := targetVal.(string); ok { - switch inputFormat { - case InputFormatJSON: - return parseJSON(valStr) - default: - return nil, fmt.Errorf("unknown inputFormat, %v", inputFormat) + if jsonStr, ok := targetVal.(string); ok { + var parsedValue map[string]interface{} + err := jsoniter.UnmarshalFromString(jsonStr, &parsedValue) + if err != nil { + return pcommon.Map{}, err + } + result := pcommon.NewMap() + for k, v := range parsedValue { + attrVal := pcommon.NewValueEmpty() + err = setValue(attrVal, v) + if err != nil { + return pcommon.Map{}, err + } + attrVal.CopyTo(result.PutEmpty(k)) } + return result, nil } return nil, nil }, nil } -func parseJSON(jsonStr string) (pcommon.Map, error) { - var parsedValue map[string]interface{} - err := jsoniter.UnmarshalFromString(jsonStr, &parsedValue) - if err != nil { - return pcommon.Map{}, err - } - result := pcommon.NewMap() - for k, v := range parsedValue { - attrVal := pcommon.NewValueEmpty() - err = setValue(attrVal, v) - if err != nil { - return pcommon.Map{}, err - } - attrVal.CopyTo(result.PutEmpty(k)) - } - return result, nil -} - func setValue(value pcommon.Value, val interface{}) error { switch v := val.(type) { case string: diff --git a/pkg/ottl/ottlfuncs/func_parse_to_map_test.go b/pkg/ottl/ottlfuncs/func_parse_json_test.go similarity index 86% rename from pkg/ottl/ottlfuncs/func_parse_to_map_test.go rename to pkg/ottl/ottlfuncs/func_parse_json_test.go index 8ee7f2269a49..3599a131b8f9 100644 --- a/pkg/ottl/ottlfuncs/func_parse_to_map_test.go +++ b/pkg/ottl/ottlfuncs/func_parse_json_test.go @@ -24,16 +24,14 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" ) -func Test_ParseToMap(t *testing.T) { +func Test_ParseJSON(t *testing.T) { tests := []struct { - name string - target ottl.Getter[any] - inputFormat string - want func(pcommon.Map) + name string + target ottl.Getter[any] + want func(pcommon.Map) }{ { - name: "handle string", - inputFormat: "json", + name: "handle string", target: ottl.StandardGetSetter[any]{ Getter: func(ctx context.Context, tCtx any) (interface{}, error) { return `{"test":"string value"}`, nil @@ -44,8 +42,7 @@ func Test_ParseToMap(t *testing.T) { }, }, { - name: "handle bool", - inputFormat: "json", + name: "handle bool", target: ottl.StandardGetSetter[any]{ Getter: func(ctx context.Context, tCtx any) (interface{}, error) { return `{"test":true}`, nil @@ -56,8 +53,7 @@ func Test_ParseToMap(t *testing.T) { }, }, { - name: "handle int", - inputFormat: "json", + name: "handle int", target: ottl.StandardGetSetter[any]{ Getter: func(ctx context.Context, tCtx any) (interface{}, error) { return `{"test":1}`, nil @@ -68,8 +64,7 @@ func Test_ParseToMap(t *testing.T) { }, }, { - name: "handle float", - inputFormat: "json", + name: "handle float", target: ottl.StandardGetSetter[any]{ Getter: func(ctx context.Context, tCtx any) (interface{}, error) { return `{"test":1.1}`, nil @@ -80,8 +75,7 @@ func Test_ParseToMap(t *testing.T) { }, }, { - name: "handle nil", - inputFormat: "json", + name: "handle nil", target: ottl.StandardGetSetter[any]{ Getter: func(ctx context.Context, tCtx any) (interface{}, error) { return `{"test":null}`, nil @@ -92,8 +86,7 @@ func Test_ParseToMap(t *testing.T) { }, }, { - name: "handle array", - inputFormat: "json", + name: "handle array", target: ottl.StandardGetSetter[any]{ Getter: func(ctx context.Context, tCtx any) (interface{}, error) { return `{"test":["string","value"]}`, nil @@ -106,8 +99,7 @@ func Test_ParseToMap(t *testing.T) { }, }, { - name: "handle nested object", - inputFormat: "json", + name: "handle nested object", target: ottl.StandardGetSetter[any]{ Getter: func(ctx context.Context, tCtx any) (interface{}, error) { return `{"test":{"nested":"true"}}`, nil @@ -119,8 +111,7 @@ func Test_ParseToMap(t *testing.T) { }, }, { - name: "updates existing", - inputFormat: "json", + name: "updates existing", target: ottl.StandardGetSetter[any]{ Getter: func(ctx context.Context, tCtx any) (interface{}, error) { return `{"existing":"pass"}`, nil @@ -131,8 +122,7 @@ func Test_ParseToMap(t *testing.T) { }, }, { - name: "complex", - inputFormat: "json", + name: "complex", target: ottl.StandardGetSetter[any]{ Getter: func(ctx context.Context, tCtx any) (interface{}, error) { return `{"test1":{"nested":"true"},"test2":"string","test3":1,"test4":1.1,"test5":[[1], [2, 3],[]],"test6":null}`, nil @@ -157,7 +147,7 @@ func Test_ParseToMap(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - exprFunc, err := ParseToMap(tt.target, "json") + exprFunc, err := ParseJSON(tt.target) assert.NoError(t, err) result, err := exprFunc(context.Background(), nil) From 1231e95bc8b6795cdb913a5e13b5a07ccfef0570 Mon Sep 17 00:00:00 2001 From: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> Date: Mon, 28 Nov 2022 13:47:57 -0700 Subject: [PATCH 05/13] Update .chloggen/ottl-parse-to-map.yaml Co-authored-by: Evan Bradley --- .chloggen/ottl-parse-to-map.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.chloggen/ottl-parse-to-map.yaml b/.chloggen/ottl-parse-to-map.yaml index 5f9d1a72bb26..0a178514a07f 100755 --- a/.chloggen/ottl-parse-to-map.yaml +++ b/.chloggen/ottl-parse-to-map.yaml @@ -5,7 +5,7 @@ change_type: enhancement component: pkg/ottl # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). -note: Add new `ParseToMap` function that can convert a json string into `pcommon.Map`. +note: Add new `ParseJSON` function that can convert a json string into `pcommon.Map`. # One or more tracking issues related to the change issues: [16444] From 4d0294f780177eb334a83b65832f9c8e8c396497 Mon Sep 17 00:00:00 2001 From: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> Date: Mon, 28 Nov 2022 13:48:55 -0700 Subject: [PATCH 06/13] Update example --- pkg/ottl/ottlfuncs/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/ottl/ottlfuncs/README.md b/pkg/ottl/ottlfuncs/README.md index 60ad12ed802e..bb955a30e750 100644 --- a/pkg/ottl/ottlfuncs/README.md +++ b/pkg/ottl/ottlfuncs/README.md @@ -149,13 +149,13 @@ JSON objects -> map[string]any Examples: -- `ParseToMap("{\"attr\":true}")` +- `ParseJSON("{\"attr\":true}")` -- `ParseToMap(attributes["kubernetes"])` +- `ParseJSON(attributes["kubernetes"])` -- `ParseToMap(body)` +- `ParseJSON(body)` ### SpanID From d1defec3899abb907ff71998f3e46245a7d871e8 Mon Sep 17 00:00:00 2001 From: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> Date: Mon, 28 Nov 2022 16:36:19 -0700 Subject: [PATCH 07/13] Apply feedback --- pkg/ottl/ottlfuncs/func_parse_json.go | 53 ++++++--------------------- 1 file changed, 11 insertions(+), 42 deletions(-) diff --git a/pkg/ottl/ottlfuncs/func_parse_json.go b/pkg/ottl/ottlfuncs/func_parse_json.go index d5325274a47c..e536fcf2feb2 100644 --- a/pkg/ottl/ottlfuncs/func_parse_json.go +++ b/pkg/ottl/ottlfuncs/func_parse_json.go @@ -16,14 +16,21 @@ package ottlfuncs // import "github.com/open-telemetry/opentelemetry-collector-c import ( "context" - "fmt" - jsoniter "github.com/json-iterator/go" "go.opentelemetry.io/collector/pdata/pcommon" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" ) +// ParseJSON factory function returns a `pcommon.Map` struct that is a result of parsing the target string as JSON +// Each JSON type is converted into a `pdata.Value` using the following map: +// +// JSON boolean -> bool +// JSON number -> float64 +// JSON string -> string +// JSON null -> nil +// JSON arrays -> pdata.SliceValue +// JSON objects -> map[string]any func ParseJSON[K any](target ottl.Getter[K]) (ottl.ExprFunc[K], error) { return func(ctx context.Context, tCtx K) (interface{}, error) { targetVal, err := target.Get(ctx, tCtx) @@ -38,52 +45,14 @@ func ParseJSON[K any](target ottl.Getter[K]) (ottl.ExprFunc[K], error) { } result := pcommon.NewMap() for k, v := range parsedValue { - attrVal := pcommon.NewValueEmpty() - err = setValue(attrVal, v) + attrVal := result.PutEmpty(k) + err = attrVal.FromRaw(v) if err != nil { return pcommon.Map{}, err } - attrVal.CopyTo(result.PutEmpty(k)) } return result, nil } return nil, nil }, nil } - -func setValue(value pcommon.Value, val interface{}) error { - switch v := val.(type) { - case string: - value.SetStr(v) - case bool: - value.SetBool(v) - case float64: - value.SetDouble(v) - case nil: - case []interface{}: - emptySlice := value.SetEmptySlice() - err := setSlice(emptySlice, v) - if err != nil { - return err - } - case map[string]interface{}: - err := value.SetEmptyMap().FromRaw(v) - if err != nil { - return err - } - default: - return fmt.Errorf("unknown type, %T", v) - } - return nil -} - -func setSlice(slice pcommon.Slice, value []interface{}) error { - for _, item := range value { - emptyValue := slice.AppendEmpty() - err := setValue(emptyValue, item) - if err != nil { - return err - } - } - return nil -} From 13afcdfdff0e35af3448a9f2476232d2a71d416d Mon Sep 17 00:00:00 2001 From: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> Date: Mon, 28 Nov 2022 16:53:37 -0700 Subject: [PATCH 08/13] Reduce further --- pkg/ottl/ottlfuncs/func_parse_json.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/pkg/ottl/ottlfuncs/func_parse_json.go b/pkg/ottl/ottlfuncs/func_parse_json.go index e536fcf2feb2..8a00532dbd77 100644 --- a/pkg/ottl/ottlfuncs/func_parse_json.go +++ b/pkg/ottl/ottlfuncs/func_parse_json.go @@ -44,14 +44,8 @@ func ParseJSON[K any](target ottl.Getter[K]) (ottl.ExprFunc[K], error) { return pcommon.Map{}, err } result := pcommon.NewMap() - for k, v := range parsedValue { - attrVal := result.PutEmpty(k) - err = attrVal.FromRaw(v) - if err != nil { - return pcommon.Map{}, err - } - } - return result, nil + err = result.FromRaw(parsedValue) + return result, err } return nil, nil }, nil From b9aa469d4eb6995e94c5d399fa2af110eeba629b Mon Sep 17 00:00:00 2001 From: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> Date: Mon, 28 Nov 2022 18:11:15 -0700 Subject: [PATCH 09/13] Update pkg/ottl/ottlfuncs/func_parse_json.go Co-authored-by: Bogdan Drutu --- pkg/ottl/ottlfuncs/func_parse_json.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ottl/ottlfuncs/func_parse_json.go b/pkg/ottl/ottlfuncs/func_parse_json.go index 8a00532dbd77..565f11ff778f 100644 --- a/pkg/ottl/ottlfuncs/func_parse_json.go +++ b/pkg/ottl/ottlfuncs/func_parse_json.go @@ -41,7 +41,7 @@ func ParseJSON[K any](target ottl.Getter[K]) (ottl.ExprFunc[K], error) { var parsedValue map[string]interface{} err := jsoniter.UnmarshalFromString(jsonStr, &parsedValue) if err != nil { - return pcommon.Map{}, err + return nil, err } result := pcommon.NewMap() err = result.FromRaw(parsedValue) From 1abb7ea7095c957e780c22edad33a974529bb933 Mon Sep 17 00:00:00 2001 From: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> Date: Mon, 28 Nov 2022 18:38:50 -0700 Subject: [PATCH 10/13] Fix lint --- pkg/ottl/ottlfuncs/func_parse_json.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/ottl/ottlfuncs/func_parse_json.go b/pkg/ottl/ottlfuncs/func_parse_json.go index 565f11ff778f..bf13d718647f 100644 --- a/pkg/ottl/ottlfuncs/func_parse_json.go +++ b/pkg/ottl/ottlfuncs/func_parse_json.go @@ -16,6 +16,7 @@ package ottlfuncs // import "github.com/open-telemetry/opentelemetry-collector-c import ( "context" + jsoniter "github.com/json-iterator/go" "go.opentelemetry.io/collector/pdata/pcommon" From 72647d23f43f17fad010eb0557cf9040bf66d0ba Mon Sep 17 00:00:00 2001 From: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> Date: Tue, 29 Nov 2022 08:20:08 -0700 Subject: [PATCH 11/13] Return error --- pkg/ottl/ottlfuncs/func_parse_json.go | 3 ++- pkg/ottl/ottlfuncs/func_parse_json_test.go | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/pkg/ottl/ottlfuncs/func_parse_json.go b/pkg/ottl/ottlfuncs/func_parse_json.go index bf13d718647f..b03144976235 100644 --- a/pkg/ottl/ottlfuncs/func_parse_json.go +++ b/pkg/ottl/ottlfuncs/func_parse_json.go @@ -16,6 +16,7 @@ package ottlfuncs // import "github.com/open-telemetry/opentelemetry-collector-c import ( "context" + "fmt" jsoniter "github.com/json-iterator/go" "go.opentelemetry.io/collector/pdata/pcommon" @@ -48,6 +49,6 @@ func ParseJSON[K any](target ottl.Getter[K]) (ottl.ExprFunc[K], error) { err = result.FromRaw(parsedValue) return result, err } - return nil, nil + return nil, fmt.Errorf("target must be a string but got %T", targetVal) }, nil } diff --git a/pkg/ottl/ottlfuncs/func_parse_json_test.go b/pkg/ottl/ottlfuncs/func_parse_json_test.go index 3599a131b8f9..5977601351a1 100644 --- a/pkg/ottl/ottlfuncs/func_parse_json_test.go +++ b/pkg/ottl/ottlfuncs/func_parse_json_test.go @@ -171,3 +171,15 @@ func Test_ParseJSON(t *testing.T) { }) } } + +func Test_ParseJSON_Error(t *testing.T) { + target := &ottl.StandardGetSetter[interface{}]{ + Getter: func(ctx context.Context, tCtx interface{}) (interface{}, error) { + return 1, nil + }, + } + exprFunc, err := ParseJSON[interface{}](target) + assert.NoError(t, err) + _, err = exprFunc(context.Background(), nil) + assert.Error(t, err) +} From eef6788ffd0ff3e6334ad0dfe301b77d87df2455 Mon Sep 17 00:00:00 2001 From: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> Date: Tue, 29 Nov 2022 08:56:28 -0700 Subject: [PATCH 12/13] Update pkg/ottl/ottlfuncs/func_parse_json.go Co-authored-by: Bogdan Drutu --- pkg/ottl/ottlfuncs/func_parse_json.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/pkg/ottl/ottlfuncs/func_parse_json.go b/pkg/ottl/ottlfuncs/func_parse_json.go index b03144976235..b3feea8ce368 100644 --- a/pkg/ottl/ottlfuncs/func_parse_json.go +++ b/pkg/ottl/ottlfuncs/func_parse_json.go @@ -39,16 +39,17 @@ func ParseJSON[K any](target ottl.Getter[K]) (ottl.ExprFunc[K], error) { if err != nil { return nil, err } - if jsonStr, ok := targetVal.(string); ok { - var parsedValue map[string]interface{} - err := jsoniter.UnmarshalFromString(jsonStr, &parsedValue) - if err != nil { - return nil, err - } - result := pcommon.NewMap() - err = result.FromRaw(parsedValue) - return result, err + jsonStr, ok := targetVal.(string) + if !ok { + return nil, fmt.Errorf("target must be a string but got %T", targetVal) } - return nil, fmt.Errorf("target must be a string but got %T", targetVal) + var parsedValue map[string]interface{} + err := jsoniter.UnmarshalFromString(jsonStr, &parsedValue) + if err != nil { + return nil, err + } + result := pcommon.NewMap() + err = result.FromRaw(parsedValue) + return result, err }, nil } From b412cf189d1813dddc69226218e9cfae7921abe4 Mon Sep 17 00:00:00 2001 From: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> Date: Tue, 29 Nov 2022 09:40:11 -0700 Subject: [PATCH 13/13] Fix --- pkg/ottl/ottlfuncs/func_parse_json.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ottl/ottlfuncs/func_parse_json.go b/pkg/ottl/ottlfuncs/func_parse_json.go index b3feea8ce368..bfb03506fc5a 100644 --- a/pkg/ottl/ottlfuncs/func_parse_json.go +++ b/pkg/ottl/ottlfuncs/func_parse_json.go @@ -44,7 +44,7 @@ func ParseJSON[K any](target ottl.Getter[K]) (ottl.ExprFunc[K], error) { return nil, fmt.Errorf("target must be a string but got %T", targetVal) } var parsedValue map[string]interface{} - err := jsoniter.UnmarshalFromString(jsonStr, &parsedValue) + err = jsoniter.UnmarshalFromString(jsonStr, &parsedValue) if err != nil { return nil, err }