diff --git a/README.md b/README.md index 57fd0dc..f5736d4 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,10 @@ # struct2env -Convert between go structures to environment variables and back (for structured config <-> shell env) -There are many go packages that are doing environment to go struct config (for instance https://github.com/kelseyhightower/envconfig) but I didn't find one doing the inverse and we needed to set a bunch of environment variables for shell and other tools to get some configuration structured as JSON and Go object, so this was born. For symetry the reverse was also added (history of commit on https://github.com/fortio/dflag/pull/50/commits) +Convert between go structures to environment variables and back (for structured config <-> shell env and to kubernetes YAML env pod spec) + +There are many go packages that are doing environment to go struct config (for instance https://github.com/kelseyhightower/envconfig) but I didn't find one doing the inverse and we needed to set a bunch of environment variables for shell and other tools to get some configuration structured as JSON and Go object, so this was born. For symmetry the reverse was also added. + +A bit later the `ToYamlWithPrefix()` was also added as alternative serialization to insert in kubernetes deployment CI templates a common cluster configuration for instance. Standalone package with 0 dependencies outside of the go standard library. Developed with go 1.20 but tested with go as old as 1.17 but should works with pretty much any go version, as it only depends on reflection and strconv. @@ -22,16 +25,24 @@ type FooConfig struct { } ``` -Turns into +Turns into (from the unit tests) ```shell -TST_FOO='a -foo with $, `, " quotes, \ and '\'' quotes' +TST_FOO='a newline: +foo with $X, ` + "`backticks`" + `, " quotes and \ and '\'' in middle and end '\''' TST_BAR='42str' TST_A_SPECIAL_BLAH='42' TST_A_BOOL=true TST_HTTP_SERVER='http://localhost:8080' TST_INT_POINTER='199' -TST_FLOAT_POINTER='3.14' +TST_FLOAT_POINTER= +TST_INNER_A='inner a' +TST_INNER_B='inner b' +TST_RECURSE_HERE_INNER_A='rec a' +TST_RECURSE_HERE_INNER_B='rec b' +TST_SOME_BINARY='AAEC' +TST_DUR=3600.1 +TST_TS='1998-11-05T14:30:00Z' +export TST_FOO TST_BAR TST_A_SPECIAL_BLAH TST_A_BOOL TST_HTTP_SERVER TST_INT_POINTER TST_FLOAT_POINTER TST_INNER_A TST_INNER_B TST_RECURSE_HERE_INNER_A TST_RECURSE_HERE_INNER_B TST_SOME_BINARY TST_DUR TST_TS ``` Using @@ -40,9 +51,34 @@ kv, errs := struct2env.StructToEnvVars(foo) txt := struct2env.ToShellWithPrefix("TST_", kv) ``` +Or + +```yaml +Y_FOO: "a newline:\nfoo with $X, `backticks`, \" quotes and \\ and ' in middle and end '" +Y_BAR: "42str" +Y_A_SPECIAL_BLAH: "42" +Y_A_BOOL: true +Y_HTTP_SERVER: "http://localhost:8080" +Y_INT_POINTER: "199" +Y_FLOAT_POINTER: null +Y_INNER_A: "inner a" +Y_INNER_B: "inner b" +Y_RECURSE_HERE_INNER_A: "rec a" +Y_RECURSE_HERE_INNER_B: "rec b" +Y_SOME_BINARY: 'AAEC' +Y_DUR: 3600.1 +Y_TS: "1998-11-05T14:30:00Z" +``` + +using +```go +kv, errs := struct2env.StructToEnvVars(foo) +txt := struct2env.ToYamlWithPrefix("Y_", kv) +``` + Type conversions: -- Most primitive type to their string representation, single quote (') escaped. +- Most primitive type to their string representation, single quote (') escaped for shell and double quote (") for YAML. - []byte are encoded as base64 - time.Time are formatted as RFC3339 - time.Duration are in (floating point) seconds. diff --git a/env.go b/env.go index 02971cc..898c544 100644 --- a/env.go +++ b/env.go @@ -94,8 +94,9 @@ func CamelCaseToLowerKebabCase(s string) string { // reoccurence of bugs like https://en.wikipedia.org/wiki/Shellshock_(software_bug) notwithstanding) // So avoid or scrub external values if possible (or use []byte type which base64 encodes the values). type KeyValue struct { - Key string // Must be safe (is when coming from Go struct names but could be bad with env:). - QuotedValue string // (Must be) Already quoted/escaped. + Key string // Must be safe (is when coming from Go struct names but could be bad with env:). + ShellQuotedVal string // (Must be) Already quoted/escaped ('' style). + YamlQuotedVal string // (Must be) Already quoted/escaped for yaml ("" with \ style). } // Escape characters such as the result string can be embedded as a single argument in a shell fragment @@ -111,8 +112,16 @@ func ShellQuote(input string) (string, error) { return "'" + strings.ReplaceAll(input, "'", `'\''`) + "'", nil } -func (kv KeyValue) String() string { - return fmt.Sprintf("%s=%s", kv.Key, kv.QuotedValue) +func YamlQuote(input string) string { + return strconv.Quote(input) +} + +func (kv KeyValue) ToShell() string { + return fmt.Sprintf("%s=%s", kv.Key, kv.ShellQuotedVal) +} + +func (kv KeyValue) ToYaml() string { + return fmt.Sprintf("%s: %s", kv.Key, kv.YamlQuotedVal) } func ToShell(kvl []KeyValue) string { @@ -125,7 +134,7 @@ func ToShellWithPrefix(prefix string, kvl []KeyValue) string { keys := make([]string, 0, len(kvl)) for _, kv := range kvl { sb.WriteString(prefix) - sb.WriteString(kv.String()) + sb.WriteString(kv.ToShell()) sb.WriteRune('\n') keys = append(keys, prefix+kv.Key) } @@ -135,22 +144,45 @@ func ToShellWithPrefix(prefix string, kvl []KeyValue) string { return sb.String() } -func SerializeValue(value interface{}) (string, error) { +func ToYamlWithPrefix(prefix string, kvl []KeyValue) string { + var sb strings.Builder + for _, kv := range kvl { + sb.WriteString(prefix) + sb.WriteString(kv.ToYaml()) + sb.WriteRune('\n') + } + return sb.String() +} + +func SerializeValue(result *KeyValue, value interface{}) error { + var err error switch v := value.(type) { case bool: res := "false" if v { res = "true" } - return res, nil + result.ShellQuotedVal = res + result.YamlQuotedVal = res + return nil case []byte: - return ShellQuote(base64.StdEncoding.EncodeToString(v)) + result.ShellQuotedVal, err = ShellQuote(base64.StdEncoding.EncodeToString(v)) + result.YamlQuotedVal = result.ShellQuotedVal // same single quoting works for yaml when no special chars is in + return err case string: - return ShellQuote(v) + result.ShellQuotedVal, err = ShellQuote(v) + result.YamlQuotedVal = YamlQuote(v) + return err case time.Duration: - return fmt.Sprintf("%g", v.Seconds()), nil + str := fmt.Sprintf("%g", v.Seconds()) + result.ShellQuotedVal = str + result.YamlQuotedVal = str + return nil default: - return ShellQuote(fmt.Sprint(value)) + str := fmt.Sprint(value) + result.ShellQuotedVal, err = ShellQuote(str) + result.YamlQuotedVal = YamlQuote(str) + return err } } @@ -196,30 +228,32 @@ func structToEnvVars(envVars []KeyValue, allErrors []error, prefix string, s int tag = CamelCaseToUpperSnakeCase(fieldType.Name) } fieldValue := v.Field(i) - stringValue := "" var err error + res := KeyValue{Key: prefix + tag} if fieldValue.Type() == reflect.TypeOf(time.Time{}) { // other wise we hit the "struct" case below timeField := fieldValue.Interface().(time.Time) - stringValue, err = SerializeValue(timeField.Format(time.RFC3339)) + err = SerializeValue(&res, timeField.Format(time.RFC3339)) if err != nil { allErrors = append(allErrors, err) } else { - envVars = append(envVars, KeyValue{Key: prefix + tag, QuotedValue: stringValue}) + envVars = append(envVars, res) } continue // Continue to the next field } switch fieldValue.Kind() { //nolint: exhaustive // we have default: for the other cases case reflect.Ptr: - if !fieldValue.IsNil() { + if fieldValue.IsNil() { + res.YamlQuotedVal = "null" + } else { fieldValue = fieldValue.Elem() - stringValue, err = SerializeValue(fieldValue.Interface()) + err = SerializeValue(&res, fieldValue.Interface()) } case reflect.Map, reflect.Array, reflect.Chan, reflect.Slice: // From that list of other types, only support []byte if fieldValue.Type().Elem().Kind() == reflect.Uint8 { - stringValue, err = SerializeValue(fieldValue.Interface()) + err = SerializeValue(&res, fieldValue.Interface()) } else { // log.LogVf("Skipping field %s of type %v, not supported", fieldType.Name, fieldType.Type) continue @@ -233,10 +267,10 @@ func structToEnvVars(envVars []KeyValue, allErrors []error, prefix string, s int err = fmt.Errorf("can't interface %s", fieldType.Name) } else { value := fieldValue.Interface() - stringValue, err = SerializeValue(value) + err = SerializeValue(&res, value) } } - envVars = append(envVars, KeyValue{Key: prefix + tag, QuotedValue: stringValue}) + envVars = append(envVars, res) if err != nil { allErrors = append(allErrors, err) } diff --git a/env_test.go b/env_test.go index cc2402f..5efd9e6 100644 --- a/env_test.go +++ b/env_test.go @@ -176,6 +176,26 @@ TST_SOME_BINARY='AAEC' TST_DUR=3600.1 TST_TS='1998-11-05T14:30:00Z' export TST_FOO TST_BAR TST_A_SPECIAL_BLAH TST_A_BOOL TST_HTTP_SERVER TST_INT_POINTER TST_FLOAT_POINTER TST_INNER_A TST_INNER_B TST_RECURSE_HERE_INNER_A TST_RECURSE_HERE_INNER_B TST_SOME_BINARY TST_DUR TST_TS +` + if str != expected { + t.Errorf("\n---expected:---\n%s\n---got:---\n%s", expected, str) + } + // YAML check + str = ToYamlWithPrefix("Y_", envVars) + expected = `Y_FOO: "a newline:\nfoo with $X, ` + "`backticks`" + `, \" quotes and \\ and ' in middle and end '" +Y_BAR: "42str" +Y_A_SPECIAL_BLAH: "42" +Y_A_BOOL: true +Y_HTTP_SERVER: "http://localhost:8080" +Y_INT_POINTER: "199" +Y_FLOAT_POINTER: null +Y_INNER_A: "inner a" +Y_INNER_B: "inner b" +Y_RECURSE_HERE_INNER_A: "rec a" +Y_RECURSE_HERE_INNER_B: "rec b" +Y_SOME_BINARY: 'AAEC' +Y_DUR: 3600.1 +Y_TS: "1998-11-05T14:30:00Z" ` if str != expected { t.Errorf("\n---expected:---\n%s\n---got:---\n%s", expected, str) @@ -192,9 +212,10 @@ export TST_FOO TST_BAR TST_A_SPECIAL_BLAH TST_A_BOOL TST_HTTP_SERVER TST_INT_POI if envVars[0].Key != "FOO" { t.Errorf("Expecting key to be present %v", envVars) } - if envVars[0].QuotedValue != "" { + if envVars[0].ShellQuotedVal != "" { t.Errorf("Expecting value to be empty %v", envVars) } + } func TestSetFromEnv(t *testing.T) {