Skip to content

Commit

Permalink
Add an expand provider, currently expands only env variables (#4098)
Browse files Browse the repository at this point in the history 
Signed-off-by: Bogdan Drutu <[email protected]>
  • Loading branch information
@bogdandrutu
bogdandrutu authored Sep 28, 2021
1 parent 5c8f7f9 commit e43250e
Show file tree
Hide file tree
Showing 13 changed files with 249 additions and 507 deletions.
39 changes: 0 additions & 39 deletions config/configunmarshaler/defaultunmarshaler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,6 @@ func unmarshalExtensions(exts map[string]map[string]interface{}, factories map[c
// Iterate over extensions and create a config for each.
for key, value := range exts {
componentConfig := config.NewMapFromStringMap(value)
expandEnvConfig(componentConfig)

// Decode the key into type and fullName components.
id, err := config.NewIDFromString(key)
Expand Down Expand Up @@ -301,7 +300,6 @@ func unmarshalReceivers(recvs map[string]map[string]interface{}, factories map[c
// Iterate over input map and create a config for each.
for key, value := range recvs {
componentConfig := config.NewMapFromStringMap(value)
expandEnvConfig(componentConfig)

// Decode the key into type and fullName components.
id, err := config.NewIDFromString(key)
Expand Down Expand Up @@ -338,7 +336,6 @@ func unmarshalExporters(exps map[string]map[string]interface{}, factories map[co
// Iterate over Exporters and create a config for each.
for key, value := range exps {
componentConfig := config.NewMapFromStringMap(value)
expandEnvConfig(componentConfig)

// Decode the key into type and fullName components.
id, err := config.NewIDFromString(key)
Expand Down Expand Up @@ -380,7 +377,6 @@ func unmarshalProcessors(procs map[string]map[string]interface{}, factories map[
// Iterate over processors and create a config for each.
for key, value := range procs {
componentConfig := config.NewMapFromStringMap(value)
expandEnvConfig(componentConfig)

// Decode the key into type and fullName components.
id, err := config.NewIDFromString(key)
Expand Down Expand Up @@ -474,41 +470,6 @@ func parseIDNames(pipelineID config.ComponentID, componentType string, names []s
return ret, nil
}

// expandEnvConfig updates a config.Map with expanded values for all the values (simple, list or map value).
// It does not expand the keys.
func expandEnvConfig(v *config.Map) {
for _, k := range v.AllKeys() {
v.Set(k, expandStringValues(v.Get(k)))
}
}

func expandStringValues(value interface{}) interface{} {
switch v := value.(type) {
default:
return v
case string:
return expandEnv(v)
case []interface{}:
nslice := make([]interface{}, 0, len(v))
for _, vint := range v {
nslice = append(nslice, expandStringValues(vint))
}
return nslice
case map[string]interface{}:
nmap := make(map[interface{}]interface{}, len(v))
for k, vint := range v {
nmap[k] = expandStringValues(vint)
}
return nmap
case map[interface{}]interface{}:
nmap := make(map[interface{}]interface{}, len(v))
for k, vint := range v {
nmap[k] = expandStringValues(vint)
}
return nmap
}
}

// expandEnvLoadedConfig is a utility function that goes recursively through a config object
// and tries to expand environment variables in its string fields.
func expandEnvLoadedConfig(s interface{}) {
Expand Down
252 changes: 0 additions & 252 deletions config/configunmarshaler/defaultunmarshaler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,258 +119,6 @@ func TestDecodeConfig(t *testing.T) {
"Did not load pipeline config correctly")
}

func TestSimpleConfig(t *testing.T) {
var testCases = []struct {
name string // test case name (also file name containing config yaml)
}{
{name: "simple-config-with-no-env"},
{name: "simple-config-with-partial-env"},
{name: "simple-config-with-all-env"},
}

const extensionExtra = "some extension string"
const extensionExtraMapValue = "some extension map value"
const extensionExtraListElement = "some extension list value"
assert.NoError(t, os.Setenv("EXTENSIONS_EXAMPLEEXTENSION_EXTRA", extensionExtra))
assert.NoError(t, os.Setenv("EXTENSIONS_EXAMPLEEXTENSION_EXTRA_MAP_EXT_VALUE_1", extensionExtraMapValue+"_1"))
assert.NoError(t, os.Setenv("EXTENSIONS_EXAMPLEEXTENSION_EXTRA_MAP_EXT_VALUE_2", extensionExtraMapValue+"_2"))
assert.NoError(t, os.Setenv("EXTENSIONS_EXAMPLEEXTENSION_EXTRA_LIST_VALUE_1", extensionExtraListElement+"_1"))
assert.NoError(t, os.Setenv("EXTENSIONS_EXAMPLEEXTENSION_EXTRA_LIST_VALUE_2", extensionExtraListElement+"_2"))

const receiverExtra = "some receiver string"
const receiverExtraMapValue = "some receiver map value"
const receiverExtraListElement = "some receiver list value"
assert.NoError(t, os.Setenv("RECEIVERS_EXAMPLERECEIVER_EXTRA", receiverExtra))
assert.NoError(t, os.Setenv("RECEIVERS_EXAMPLERECEIVER_EXTRA_MAP_RECV_VALUE_1", receiverExtraMapValue+"_1"))
assert.NoError(t, os.Setenv("RECEIVERS_EXAMPLERECEIVER_EXTRA_MAP_RECV_VALUE_2", receiverExtraMapValue+"_2"))
assert.NoError(t, os.Setenv("RECEIVERS_EXAMPLERECEIVER_EXTRA_LIST_VALUE_1", receiverExtraListElement+"_1"))
assert.NoError(t, os.Setenv("RECEIVERS_EXAMPLERECEIVER_EXTRA_LIST_VALUE_2", receiverExtraListElement+"_2"))

const processorExtra = "some processor string"
const processorExtraMapValue = "some processor map value"
const processorExtraListElement = "some processor list value"
assert.NoError(t, os.Setenv("PROCESSORS_EXAMPLEPROCESSOR_EXTRA", processorExtra))
assert.NoError(t, os.Setenv("PROCESSORS_EXAMPLEPROCESSOR_EXTRA_MAP_PROC_VALUE_1", processorExtraMapValue+"_1"))
assert.NoError(t, os.Setenv("PROCESSORS_EXAMPLEPROCESSOR_EXTRA_MAP_PROC_VALUE_2", processorExtraMapValue+"_2"))
assert.NoError(t, os.Setenv("PROCESSORS_EXAMPLEPROCESSOR_EXTRA_LIST_VALUE_1", processorExtraListElement+"_1"))
assert.NoError(t, os.Setenv("PROCESSORS_EXAMPLEPROCESSOR_EXTRA_LIST_VALUE_2", processorExtraListElement+"_2"))

const exporterExtra = "some exporter string"
const exporterExtraMapValue = "some exporter map value"
const exporterExtraListElement = "some exporter list value"
assert.NoError(t, os.Setenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA_INT", "65"))
assert.NoError(t, os.Setenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA", exporterExtra))
assert.NoError(t, os.Setenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA_MAP_EXP_VALUE_1", exporterExtraMapValue+"_1"))
assert.NoError(t, os.Setenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA_MAP_EXP_VALUE_2", exporterExtraMapValue+"_2"))
assert.NoError(t, os.Setenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA_LIST_VALUE_1", exporterExtraListElement+"_1"))
assert.NoError(t, os.Setenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA_LIST_VALUE_2", exporterExtraListElement+"_2"))

defer func() {
assert.NoError(t, os.Unsetenv("EXTENSIONS_EXAMPLEEXTENSION_EXTRA"))
assert.NoError(t, os.Unsetenv("EXTENSIONS_EXAMPLEEXTENSION_EXTRA_MAP_EXT_VALUE"))
assert.NoError(t, os.Unsetenv("EXTENSIONS_EXAMPLEEXTENSION_EXTRA_LIST_VALUE_1"))

assert.NoError(t, os.Unsetenv("RECEIVERS_EXAMPLERECEIVER_EXTRA"))
assert.NoError(t, os.Unsetenv("RECEIVERS_EXAMPLERECEIVER_EXTRA_MAP_RECV_VALUE"))
assert.NoError(t, os.Unsetenv("RECEIVERS_EXAMPLERECEIVER_EXTRA_LIST_VALUE_1"))

assert.NoError(t, os.Unsetenv("PROCESSORS_EXAMPLEPROCESSOR_EXTRA"))
assert.NoError(t, os.Unsetenv("PROCESSORS_EXAMPLEPROCESSOR_EXTRA_MAP_PROC_VALUE"))
assert.NoError(t, os.Unsetenv("PROCESSORS_EXAMPLEPROCESSOR_EXTRA_LIST_VALUE_1"))

assert.NoError(t, os.Unsetenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA_INT"))
assert.NoError(t, os.Unsetenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA"))
assert.NoError(t, os.Unsetenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA_MAP_EXP_VALUE"))
assert.NoError(t, os.Unsetenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA_LIST_VALUE_1"))
}()

for _, test := range testCases {
t.Run(test.name, func(t *testing.T) {
factories, err := testcomponents.ExampleComponents()
assert.NoError(t, err)

// Unmarshal the config
cfg, err := loadConfigFile(t, path.Join(".", "testdata", test.name+".yaml"), factories)
require.NoError(t, err, "Unable to load config")

// Verify extensions.
assert.Equalf(t, 1, len(cfg.Extensions), "TEST[%s]", test.name)
assert.Equalf(t,
&testcomponents.ExampleExtensionCfg{
ExtensionSettings: config.NewExtensionSettings(config.NewID("exampleextension")),
ExtraSetting: extensionExtra,
ExtraMapSetting: map[string]string{"ext-1": extensionExtraMapValue + "_1", "ext-2": extensionExtraMapValue + "_2"},
ExtraListSetting: []string{extensionExtraListElement + "_1", extensionExtraListElement + "_2"},
},
cfg.Extensions[config.NewID("exampleextension")],
"TEST[%s] Did not load extension config correctly", test.name)

// Verify service.
assert.Equalf(t, 1, len(cfg.Service.Extensions), "TEST[%s]", test.name)
assert.Equalf(t, config.NewID("exampleextension"), cfg.Service.Extensions[0], "TEST[%s]", test.name)

// Verify receivers
assert.Equalf(t, 1, len(cfg.Receivers), "TEST[%s]", test.name)

assert.Equalf(t,
&testcomponents.ExampleReceiver{
ReceiverSettings: config.NewReceiverSettings(config.NewID("examplereceiver")),
TCPAddr: confignet.TCPAddr{
Endpoint: "localhost:1234",
},
ExtraSetting: receiverExtra,
ExtraMapSetting: map[string]string{"recv.1": receiverExtraMapValue + "_1", "recv.2": receiverExtraMapValue + "_2"},
ExtraListSetting: []string{receiverExtraListElement + "_1", receiverExtraListElement + "_2"},
},
cfg.Receivers[config.NewID("examplereceiver")],
"TEST[%s] Did not load receiver config correctly", test.name)

// Verify exporters
assert.Equalf(t, 1, len(cfg.Exporters), "TEST[%s]", test.name)

assert.Equalf(t,
&testcomponents.ExampleExporter{
ExporterSettings: config.NewExporterSettings(config.NewID("exampleexporter")),
ExtraInt: 65,
ExtraSetting: exporterExtra,
ExtraMapSetting: map[string]string{"exp_1": exporterExtraMapValue + "_1", "exp_2": exporterExtraMapValue + "_2"},
ExtraListSetting: []string{exporterExtraListElement + "_1", exporterExtraListElement + "_2"},
},
cfg.Exporters[config.NewID("exampleexporter")],
"TEST[%s] Did not load exporter config correctly", test.name)

// Verify Processors
assert.Equalf(t, 1, len(cfg.Processors), "TEST[%s]", test.name)

assert.Equalf(t,
&testcomponents.ExampleProcessorCfg{
ProcessorSettings: config.NewProcessorSettings(config.NewID("exampleprocessor")),
ExtraSetting: processorExtra,
ExtraMapSetting: map[string]string{"proc_1": processorExtraMapValue + "_1", "proc_2": processorExtraMapValue + "_2"},
ExtraListSetting: []string{processorExtraListElement + "_1", processorExtraListElement + "_2"},
},
cfg.Processors[config.NewID("exampleprocessor")],
"TEST[%s] Did not load processor config correctly", test.name)

// Verify Pipelines
assert.Equalf(t, 1, len(cfg.Service.Pipelines), "TEST[%s]", test.name)

assert.Equalf(t,
&config.Pipeline{
Name: "traces",
InputType: config.TracesDataType,
Receivers: []config.ComponentID{config.NewID("examplereceiver")},
Processors: []config.ComponentID{config.NewID("exampleprocessor")},
Exporters: []config.ComponentID{config.NewID("exampleexporter")},
},
cfg.Service.Pipelines["traces"],
"TEST[%s] Did not load pipeline config correctly", test.name)
})
}
}

func TestEscapedEnvVars(t *testing.T) {
const receiverExtraMapValue = "some receiver map value"
assert.NoError(t, os.Setenv("RECEIVERS_EXAMPLERECEIVER_EXTRA_MAP_RECV_VALUE_2", receiverExtraMapValue))
defer func() {
assert.NoError(t, os.Unsetenv("RECEIVERS_EXAMPLERECEIVER_EXTRA_MAP_RECV_VALUE_2"))
}()

factories, err := testcomponents.ExampleComponents()
assert.NoError(t, err)

// Unmarshal the config
cfg, err := loadConfigFile(t, path.Join(".", "testdata", "simple-config-with-escaped-env.yaml"), factories)
require.NoError(t, err, "Unable to load config")

// Verify extensions.
assert.Equal(t, 1, len(cfg.Extensions))
assert.Equal(t,
&testcomponents.ExampleExtensionCfg{
ExtensionSettings: config.NewExtensionSettings(config.NewID("exampleextension")),
ExtraSetting: "${EXTENSIONS_EXAMPLEEXTENSION_EXTRA}",
ExtraMapSetting: map[string]string{"ext-1": "${EXTENSIONS_EXAMPLEEXTENSION_EXTRA_MAP_EXT_VALUE_1}", "ext-2": "${EXTENSIONS_EXAMPLEEXTENSION_EXTRA_MAP_EXT_VALUE_2}"},
ExtraListSetting: []string{"${EXTENSIONS_EXAMPLEEXTENSION_EXTRA_LIST_VALUE_1}", "${EXTENSIONS_EXAMPLEEXTENSION_EXTRA_LIST_VALUE_2}"},
},
cfg.Extensions[config.NewID("exampleextension")],
"Did not load extension config correctly")

// Verify service.
assert.Equal(t, 1, len(cfg.Service.Extensions))
assert.Equal(t, config.NewID("exampleextension"), cfg.Service.Extensions[0])

// Verify receivers
assert.Equal(t, 1, len(cfg.Receivers))

assert.Equal(t,
&testcomponents.ExampleReceiver{
ReceiverSettings: config.NewReceiverSettings(config.NewID("examplereceiver")),
TCPAddr: confignet.TCPAddr{
Endpoint: "localhost:1234",
},
ExtraSetting: "$RECEIVERS_EXAMPLERECEIVER_EXTRA",
ExtraMapSetting: map[string]string{
// $$ -> escaped $
"recv.1": "$RECEIVERS_EXAMPLERECEIVER_EXTRA_MAP_RECV_VALUE_1",
// $$$ -> escaped $ + substituted env var
"recv.2": "$" + receiverExtraMapValue,
// $$$$ -> two escaped $
"recv.3": "$$RECEIVERS_EXAMPLERECEIVER_EXTRA_MAP_RECV_VALUE_3",
// escaped $ in the middle
"recv.4": "some${RECEIVERS_EXAMPLERECEIVER_EXTRA_MAP_RECV_VALUE_4}text",
// $$$$ -> two escaped $
"recv.5": "${ONE}${TWO}",
// trailing escaped $
"recv.6": "text$",
// escaped $ alone
"recv.7": "$",
},
ExtraListSetting: []string{"$RECEIVERS_EXAMPLERECEIVER_EXTRA_LIST_VALUE_1", "$RECEIVERS_EXAMPLERECEIVER_EXTRA_LIST_VALUE_2"},
},
cfg.Receivers[config.NewID("examplereceiver")],
"Did not load receiver config correctly")

// Verify exporters
assert.Equal(t, 1, len(cfg.Exporters))

assert.Equal(t,
&testcomponents.ExampleExporter{
ExporterSettings: config.NewExporterSettings(config.NewID("exampleexporter")),
ExtraSetting: "${EXPORTERS_EXAMPLEEXPORTER_EXTRA}",
ExtraMapSetting: map[string]string{"exp_1": "${EXPORTERS_EXAMPLEEXPORTER_EXTRA_MAP_EXP_VALUE_1}", "exp_2": "${EXPORTERS_EXAMPLEEXPORTER_EXTRA_MAP_EXP_VALUE_2}"},
ExtraListSetting: []string{"${EXPORTERS_EXAMPLEEXPORTER_EXTRA_LIST_VALUE_1}", "${EXPORTERS_EXAMPLEEXPORTER_EXTRA_LIST_VALUE_2}"},
},
cfg.Exporters[config.NewID("exampleexporter")],
"Did not load exporter config correctly")

// Verify Processors
assert.Equal(t, 1, len(cfg.Processors))

assert.Equal(t,
&testcomponents.ExampleProcessorCfg{
ProcessorSettings: config.NewProcessorSettings(config.NewID("exampleprocessor")),
ExtraSetting: "$PROCESSORS_EXAMPLEPROCESSOR_EXTRA",
ExtraMapSetting: map[string]string{"proc_1": "$PROCESSORS_EXAMPLEPROCESSOR_EXTRA_MAP_PROC_VALUE_1", "proc_2": "$PROCESSORS_EXAMPLEPROCESSOR_EXTRA_MAP_PROC_VALUE_2"},
ExtraListSetting: []string{"$PROCESSORS_EXAMPLEPROCESSOR_EXTRA_LIST_VALUE_1", "$PROCESSORS_EXAMPLEPROCESSOR_EXTRA_LIST_VALUE_2"},
},
cfg.Processors[config.NewID("exampleprocessor")],
"Did not load processor config correctly")

// Verify Pipelines
assert.Equal(t, 1, len(cfg.Service.Pipelines))

assert.Equal(t,
&config.Pipeline{
Name: "traces",
InputType: config.TracesDataType,
Receivers: []config.ComponentID{config.NewID("examplereceiver")},
Processors: []config.ComponentID{config.NewID("exampleprocessor")},
Exporters: []config.ComponentID{config.NewID("exampleexporter")},
},
cfg.Service.Pipelines["traces"],
"Did not load pipeline config correctly")
}

func TestDecodeConfig_Invalid(t *testing.T) {

var testCases = []struct {
Expand Down
51 changes: 0 additions & 51 deletions config/configunmarshaler/testdata/simple-config-with-all-env.yaml
View file

This file was deleted.

Loading

0 comments on commit e43250e

Please sign in to comment.