From 982d52c7bfa92870eed1b0dbef02302134278b6b Mon Sep 17 00:00:00 2001 From: "N. Le Roux" Date: Mon, 8 May 2023 12:14:26 +0200 Subject: [PATCH] use compose-go to interpolate compose variables This allows for nested variables interpolation. Fixes #4265 Signed-off-by: N. Le Roux --- cli/compose/interpolation/interpolation.go | 2 +- .../interpolation/interpolation_test.go | 6 + cli/compose/loader/loader.go | 2 +- cli/compose/template/template.go | 248 --------------- cli/compose/template/template_test.go | 285 ------------------ vendor.mod | 1 + vendor.sum | 2 + 7 files changed, 11 insertions(+), 535 deletions(-) delete mode 100644 cli/compose/template/template.go delete mode 100644 cli/compose/template/template_test.go diff --git a/cli/compose/interpolation/interpolation.go b/cli/compose/interpolation/interpolation.go index ee11656f60ac..ed4cf2871648 100644 --- a/cli/compose/interpolation/interpolation.go +++ b/cli/compose/interpolation/interpolation.go @@ -7,7 +7,7 @@ import ( "os" "strings" - "github.com/docker/cli/cli/compose/template" + "github.com/compose-spec/compose-go/v2/template" "github.com/pkg/errors" ) diff --git a/cli/compose/interpolation/interpolation_test.go b/cli/compose/interpolation/interpolation_test.go index cc7ffacabbe6..c0babf6c9a6e 100644 --- a/cli/compose/interpolation/interpolation_test.go +++ b/cli/compose/interpolation/interpolation_test.go @@ -14,6 +14,7 @@ import ( var defaults = map[string]string{ "USER": "jenny", "FOO": "bar", + "CMD": "my-cmd", "count": "5", } @@ -26,6 +27,10 @@ func TestInterpolate(t *testing.T) { services := map[string]any{ "servicea": map[string]any{ "image": "example:${USER}", + "command": []any{ + "${CMD:-${UNDEF_CMD:-}}", + "${UNDEF:-${CMD}}", + }, "volumes": []any{"$FOO:/target"}, "logging": map[string]any{ "driver": "${FOO}", @@ -38,6 +43,7 @@ func TestInterpolate(t *testing.T) { expected := map[string]any{ "servicea": map[string]any{ "image": "example:jenny", + "command": []any{"my-cmd", "my-cmd"}, "volumes": []any{"bar:/target"}, "logging": map[string]any{ "driver": "bar", diff --git a/cli/compose/loader/loader.go b/cli/compose/loader/loader.go index 7bc420b2b1e6..28dd024471e0 100644 --- a/cli/compose/loader/loader.go +++ b/cli/compose/loader/loader.go @@ -13,9 +13,9 @@ import ( "strings" "time" + "github.com/compose-spec/compose-go/v2/template" interp "github.com/docker/cli/cli/compose/interpolation" "github.com/docker/cli/cli/compose/schema" - "github.com/docker/cli/cli/compose/template" "github.com/docker/cli/cli/compose/types" "github.com/docker/cli/opts" "github.com/docker/docker/api/types/versions" diff --git a/cli/compose/template/template.go b/cli/compose/template/template.go deleted file mode 100644 index 1507c0ee6e7f..000000000000 --- a/cli/compose/template/template.go +++ /dev/null @@ -1,248 +0,0 @@ -// FIXME(thaJeztah): remove once we are a module; the go:build directive prevents go from downgrading language version to go1.16: -//go:build go1.22 - -package template - -import ( - "fmt" - "regexp" - "strings" -) - -const ( - delimiter = "\\$" - subst = "[_a-z][_a-z0-9]*(?::?[-?][^}]*)?" -) - -var defaultPattern = regexp.MustCompile(fmt.Sprintf( - "%s(?i:(?P%s)|(?P%s)|{(?P%s)}|(?P))", - delimiter, delimiter, subst, subst, -)) - -// DefaultSubstituteFuncs contains the default SubstituteFunc used by the docker cli -var DefaultSubstituteFuncs = []SubstituteFunc{ - softDefault, - hardDefault, - requiredNonEmpty, - required, -} - -// InvalidTemplateError is returned when a variable template is not in a valid -// format -type InvalidTemplateError struct { - Template string -} - -func (e InvalidTemplateError) Error() string { - return fmt.Sprintf("Invalid template: %#v", e.Template) -} - -// Mapping is a user-supplied function which maps from variable names to values. -// Returns the value as a string and a bool indicating whether -// the value is present, to distinguish between an empty string -// and the absence of a value. -type Mapping func(string) (string, bool) - -// SubstituteFunc is a user-supplied function that apply substitution. -// Returns the value as a string, a bool indicating if the function could apply -// the substitution and an error. -type SubstituteFunc func(string, Mapping) (string, bool, error) - -// SubstituteWith substitutes variables in the string with their values. -// It accepts additional substitute function. -func SubstituteWith(template string, mapping Mapping, pattern *regexp.Regexp, subsFuncs ...SubstituteFunc) (string, error) { - var err error - result := pattern.ReplaceAllStringFunc(template, func(substring string) string { - matches := pattern.FindStringSubmatch(substring) - groups := matchGroups(matches, pattern) - if escaped := groups["escaped"]; escaped != "" { - return escaped - } - - substitution := groups["named"] - if substitution == "" { - substitution = groups["braced"] - } - - if substitution == "" { - err = &InvalidTemplateError{Template: template} - return "" - } - - for _, f := range subsFuncs { - var ( - value string - applied bool - ) - value, applied, err = f(substitution, mapping) - if err != nil { - return "" - } - if !applied { - continue - } - return value - } - - value, _ := mapping(substitution) - return value - }) - - return result, err -} - -// Substitute variables in the string with their values -func Substitute(template string, mapping Mapping) (string, error) { - return SubstituteWith(template, mapping, defaultPattern, DefaultSubstituteFuncs...) -} - -// ExtractVariables returns a map of all the variables defined in the specified -// composefile (dict representation) and their default value if any. -func ExtractVariables(configDict map[string]any, pattern *regexp.Regexp) map[string]string { - if pattern == nil { - pattern = defaultPattern - } - return recurseExtract(configDict, pattern) -} - -func recurseExtract(value any, pattern *regexp.Regexp) map[string]string { - m := map[string]string{} - - switch value := value.(type) { - case string: - if values, is := extractVariable(value, pattern); is { - for _, v := range values { - m[v.name] = v.value - } - } - case map[string]any: - for _, elem := range value { - submap := recurseExtract(elem, pattern) - for key, value := range submap { - m[key] = value - } - } - - case []any: - for _, elem := range value { - if values, is := extractVariable(elem, pattern); is { - for _, v := range values { - m[v.name] = v.value - } - } - } - } - - return m -} - -type extractedValue struct { - name string - value string -} - -func extractVariable(value any, pattern *regexp.Regexp) ([]extractedValue, bool) { - sValue, ok := value.(string) - if !ok { - return []extractedValue{}, false - } - matches := pattern.FindAllStringSubmatch(sValue, -1) - if len(matches) == 0 { - return []extractedValue{}, false - } - values := []extractedValue{} - for _, match := range matches { - groups := matchGroups(match, pattern) - if escaped := groups["escaped"]; escaped != "" { - continue - } - val := groups["named"] - if val == "" { - val = groups["braced"] - } - name := val - var defaultValue string - switch { - case strings.Contains(val, ":?"): - name, _ = partition(val, ":?") - case strings.Contains(val, "?"): - name, _ = partition(val, "?") - case strings.Contains(val, ":-"): - name, defaultValue = partition(val, ":-") - case strings.Contains(val, "-"): - name, defaultValue = partition(val, "-") - } - values = append(values, extractedValue{name: name, value: defaultValue}) - } - return values, len(values) > 0 -} - -// Soft default (fall back if unset or empty) -func softDefault(substitution string, mapping Mapping) (string, bool, error) { - sep := ":-" - if !strings.Contains(substitution, sep) { - return "", false, nil - } - name, defaultValue := partition(substitution, sep) - value, ok := mapping(name) - if !ok || value == "" { - return defaultValue, true, nil - } - return value, true, nil -} - -// Hard default (fall back if-and-only-if empty) -func hardDefault(substitution string, mapping Mapping) (string, bool, error) { - sep := "-" - if !strings.Contains(substitution, sep) { - return "", false, nil - } - name, defaultValue := partition(substitution, sep) - value, ok := mapping(name) - if !ok { - return defaultValue, true, nil - } - return value, true, nil -} - -func requiredNonEmpty(substitution string, mapping Mapping) (string, bool, error) { - return withRequired(substitution, mapping, ":?", func(v string) bool { return v != "" }) -} - -func required(substitution string, mapping Mapping) (string, bool, error) { - return withRequired(substitution, mapping, "?", func(_ string) bool { return true }) -} - -func withRequired(substitution string, mapping Mapping, sep string, valid func(string) bool) (string, bool, error) { - if !strings.Contains(substitution, sep) { - return "", false, nil - } - name, errorMessage := partition(substitution, sep) - value, ok := mapping(name) - if !ok || !valid(value) { - return "", true, &InvalidTemplateError{ - Template: fmt.Sprintf("required variable %s is missing a value: %s", name, errorMessage), - } - } - return value, true, nil -} - -func matchGroups(matches []string, pattern *regexp.Regexp) map[string]string { - groups := make(map[string]string) - for i, name := range pattern.SubexpNames()[1:] { - groups[name] = matches[i+1] - } - return groups -} - -// Split the string at the first occurrence of sep, and return the part before the separator, -// and the part after the separator. -// -// If the separator is not found, return the string itself, followed by an empty string. -func partition(s, sep string) (string, string) { - k, v, ok := strings.Cut(s, sep) - if !ok { - return s, "" - } - return k, v -} diff --git a/cli/compose/template/template_test.go b/cli/compose/template/template_test.go deleted file mode 100644 index 92b95460ebf4..000000000000 --- a/cli/compose/template/template_test.go +++ /dev/null @@ -1,285 +0,0 @@ -// FIXME(thaJeztah): remove once we are a module; the go:build directive prevents go from downgrading language version to go1.16: -//go:build go1.22 - -package template - -import ( - "fmt" - "testing" - - "gotest.tools/v3/assert" - is "gotest.tools/v3/assert/cmp" -) - -var defaults = map[string]string{ - "FOO": "first", - "BAR": "", -} - -func defaultMapping(name string) (string, bool) { - val, ok := defaults[name] - return val, ok -} - -func TestEscaped(t *testing.T) { - result, err := Substitute("$${foo}", defaultMapping) - assert.NilError(t, err) - assert.Check(t, is.Equal("${foo}", result)) -} - -func TestSubstituteNoMatch(t *testing.T) { - result, err := Substitute("foo", defaultMapping) - assert.NilError(t, err) - assert.Equal(t, "foo", result) -} - -func TestInvalid(t *testing.T) { - invalidTemplates := []string{ - "${", - "$}", - "${}", - "${ }", - "${ foo}", - "${foo }", - "${foo!}", - } - - for _, template := range invalidTemplates { - _, err := Substitute(template, defaultMapping) - assert.ErrorContains(t, err, "Invalid template") - } -} - -func TestNoValueNoDefault(t *testing.T) { - for _, template := range []string{"This ${missing} var", "This ${BAR} var"} { - result, err := Substitute(template, defaultMapping) - assert.NilError(t, err) - assert.Check(t, is.Equal("This var", result)) - } -} - -func TestValueNoDefault(t *testing.T) { - for _, template := range []string{"This $FOO var", "This ${FOO} var"} { - result, err := Substitute(template, defaultMapping) - assert.NilError(t, err) - assert.Check(t, is.Equal("This first var", result)) - } -} - -func TestNoValueWithDefault(t *testing.T) { - for _, template := range []string{"ok ${missing:-def}", "ok ${missing-def}"} { - result, err := Substitute(template, defaultMapping) - assert.NilError(t, err) - assert.Check(t, is.Equal("ok def", result)) - } -} - -func TestEmptyValueWithSoftDefault(t *testing.T) { - result, err := Substitute("ok ${BAR:-def}", defaultMapping) - assert.NilError(t, err) - assert.Check(t, is.Equal("ok def", result)) -} - -func TestValueWithSoftDefault(t *testing.T) { - result, err := Substitute("ok ${FOO:-def}", defaultMapping) - assert.NilError(t, err) - assert.Check(t, is.Equal("ok first", result)) -} - -func TestEmptyValueWithHardDefault(t *testing.T) { - result, err := Substitute("ok ${BAR-def}", defaultMapping) - assert.NilError(t, err) - assert.Check(t, is.Equal("ok ", result)) -} - -func TestNonAlphanumericDefault(t *testing.T) { - result, err := Substitute("ok ${BAR:-/non:-alphanumeric}", defaultMapping) - assert.NilError(t, err) - assert.Check(t, is.Equal("ok /non:-alphanumeric", result)) -} - -func TestMandatoryVariableErrors(t *testing.T) { - testCases := []struct { - template string - expectedError string - }{ - { - template: "not ok ${UNSET_VAR:?Mandatory Variable Unset}", - expectedError: "required variable UNSET_VAR is missing a value: Mandatory Variable Unset", - }, - { - template: "not ok ${BAR:?Mandatory Variable Empty}", - expectedError: "required variable BAR is missing a value: Mandatory Variable Empty", - }, - { - template: "not ok ${UNSET_VAR:?}", - expectedError: "required variable UNSET_VAR is missing a value", - }, - { - template: "not ok ${UNSET_VAR?Mandatory Variable Unset}", - expectedError: "required variable UNSET_VAR is missing a value: Mandatory Variable Unset", - }, - { - template: "not ok ${UNSET_VAR?}", - expectedError: "required variable UNSET_VAR is missing a value", - }, - } - - for _, tc := range testCases { - _, err := Substitute(tc.template, defaultMapping) - assert.Check(t, is.ErrorContains(err, tc.expectedError)) - assert.Check(t, is.ErrorType(err, &InvalidTemplateError{})) - } -} - -func TestDefaultsForMandatoryVariables(t *testing.T) { - testCases := []struct { - template string - expected string - }{ - { - template: "ok ${FOO:?err}", - expected: "ok first", - }, - { - template: "ok ${FOO?err}", - expected: "ok first", - }, - { - template: "ok ${BAR?err}", - expected: "ok ", - }, - } - - for _, tc := range testCases { - result, err := Substitute(tc.template, defaultMapping) - assert.NilError(t, err) - assert.Check(t, is.Equal(tc.expected, result)) - } -} - -func TestSubstituteWithCustomFunc(t *testing.T) { - errIsMissing := func(substitution string, mapping Mapping) (string, bool, error) { - value, found := mapping(substitution) - if !found { - return "", true, &InvalidTemplateError{ - Template: fmt.Sprintf("required variable %s is missing a value", substitution), - } - } - return value, true, nil - } - - result, err := SubstituteWith("ok ${FOO}", defaultMapping, defaultPattern, errIsMissing) - assert.NilError(t, err) - assert.Check(t, is.Equal("ok first", result)) - - result, err = SubstituteWith("ok ${BAR}", defaultMapping, defaultPattern, errIsMissing) - assert.NilError(t, err) - assert.Check(t, is.Equal("ok ", result)) - - _, err = SubstituteWith("ok ${NOTHERE}", defaultMapping, defaultPattern, errIsMissing) - assert.Check(t, is.ErrorContains(err, "required variable")) -} - -func TestExtractVariables(t *testing.T) { - testCases := []struct { - name string - dict map[string]any - expected map[string]string - }{ - { - name: "empty", - dict: map[string]any{}, - expected: map[string]string{}, - }, - { - name: "no-variables", - dict: map[string]any{ - "foo": "bar", - }, - expected: map[string]string{}, - }, - { - name: "variable-without-curly-braces", - dict: map[string]any{ - "foo": "$bar", - }, - expected: map[string]string{ - "bar": "", - }, - }, - { - name: "variable", - dict: map[string]any{ - "foo": "${bar}", - }, - expected: map[string]string{ - "bar": "", - }, - }, - { - name: "required-variable", - dict: map[string]any{ - "foo": "${bar?:foo}", - }, - expected: map[string]string{ - "bar": "", - }, - }, - { - name: "required-variable2", - dict: map[string]any{ - "foo": "${bar?foo}", - }, - expected: map[string]string{ - "bar": "", - }, - }, - { - name: "default-variable", - dict: map[string]any{ - "foo": "${bar:-foo}", - }, - expected: map[string]string{ - "bar": "foo", - }, - }, - { - name: "default-variable2", - dict: map[string]any{ - "foo": "${bar-foo}", - }, - expected: map[string]string{ - "bar": "foo", - }, - }, - { - name: "multiple-values", - dict: map[string]any{ - "foo": "${bar:-foo}", - "bar": map[string]any{ - "foo": "${fruit:-banana}", - "bar": "vegetable", - }, - "baz": []any{ - "foo", - "$docker:${project:-cli}", - "$toto", - }, - }, - expected: map[string]string{ - "bar": "foo", - "fruit": "banana", - "toto": "", - "docker": "", - "project": "cli", - }, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - actual := ExtractVariables(tc.dict, defaultPattern) - assert.Check(t, is.DeepEqual(actual, tc.expected)) - }) - } -} diff --git a/vendor.mod b/vendor.mod index 24fa088406ff..3b119cba1dd1 100644 --- a/vendor.mod +++ b/vendor.mod @@ -8,6 +8,7 @@ go 1.22.0 require ( dario.cat/mergo v1.0.1 + github.com/compose-spec/compose-go/v2 v2.4.7 github.com/containerd/platforms v0.2.1 github.com/creack/pty v1.1.21 github.com/distribution/reference v0.6.0 diff --git a/vendor.sum b/vendor.sum index d6150b50bab4..ba6abe3e8eda 100644 --- a/vendor.sum +++ b/vendor.sum @@ -32,6 +32,8 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004/go.mod h1:yMWuSON2oQp+43nFtAV/uvKQIFpSPerB57DCt9t8sSA= github.com/cloudflare/cfssl v1.6.4 h1:NMOvfrEjFfC63K3SGXgAnFdsgkmiq4kATme5BfcqrO8= github.com/cloudflare/cfssl v1.6.4/go.mod h1:8b3CQMxfWPAeom3zBnGJ6sd+G1NkL5TXqmDXacb+1J0= +github.com/compose-spec/compose-go/v2 v2.4.7 h1:WNpz5bIbKG+G+w9pfu72B1ZXr+Og9jez8TMEo8ecXPk= +github.com/compose-spec/compose-go/v2 v2.4.7/go.mod h1:lFN0DrMxIncJGYAXTfWuajfwj5haBJqrBkarHcnjJKc= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=