Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Environment variables priority #9636

Merged
merged 2 commits into from
Jul 29, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions cmd/compose/compose.go
Original file line number Diff line number Diff line change
Expand Up @@ -354,10 +354,8 @@ func setEnvWithDotEnv(prjOpts *projectOptions) error {
return err
}
for k, v := range envFromFile {
if _, ok := os.LookupEnv(k); !ok {
if err := os.Setenv(k, v); err != nil {
return err
}
if err := os.Setenv(k, v); err != nil { // overwrite the process env with merged OS + env file results
return err
}
}
return nil
Expand Down
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -151,3 +151,5 @@ replace (
k8s.io/apimachinery => k8s.io/apimachinery v0.22.4
k8s.io/client-go => k8s.io/client-go v0.22.4
)

replace github.com/compose-spec/compose-go => github.com/compose-spec/compose-go v1.2.9-0.20220712021117-dcfe0889b601 // TODO Remove and bump require section when PR is merged
Copy link
Contributor Author

@ulyssessouza ulyssessouza Jul 12, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note that this points to compose-spec/compose-go#280
This PR shouldn't be merged before this dependency is merged and updated here.

7 changes: 2 additions & 5 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -285,9 +285,8 @@ github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h
github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
github.com/compose-spec/compose-go v1.2.1/go.mod h1:pAy7Mikpeft4pxkFU565/DRHEbDfR84G6AQuiL+Hdg8=
github.com/compose-spec/compose-go v1.2.9 h1:+7q2X8gCd16qUX+FU5EPtepK9lWZSGfc8Xe2cGmZqDc=
github.com/compose-spec/compose-go v1.2.9/go.mod h1:rGaQw1U8uhZhmANQHIocMnQ+qu6ER2+1OwhWjRqQEPI=
github.com/compose-spec/compose-go v1.2.9-0.20220712021117-dcfe0889b601 h1:dG3F1lPuUkmpxGs5qB3dFhqsXp06bpJHd0Btt6K15cU=
github.com/compose-spec/compose-go v1.2.9-0.20220712021117-dcfe0889b601/go.mod h1:rGaQw1U8uhZhmANQHIocMnQ+qu6ER2+1OwhWjRqQEPI=
github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE=
github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU=
github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=
Expand Down Expand Up @@ -1007,7 +1006,6 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.3.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
Expand Down Expand Up @@ -2121,7 +2119,6 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
gotest.tools/v3 v3.1.0/go.mod h1:fHy7eyTmJFO5bQbUsEGQ1v4m2J3Jz9eWL54TP2/ZuYQ=
gotest.tools/v3 v3.3.0 h1:MfDY1b1/0xN1CyMlQDac0ziEy9zJQd9CXBRRDHw2jJo=
gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A=
grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
Expand Down
9 changes: 3 additions & 6 deletions pkg/compose/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,15 +114,12 @@ func applyRunOptions(project *types.Project, service *types.ServiceConfig, opts
service.Entrypoint = opts.Entrypoint
}
if len(opts.Environment) > 0 {
env := types.NewMappingWithEquals(opts.Environment)
projectEnv := env.Resolve(func(s string) (string, bool) {
if _, ok := service.Environment[s]; ok {
return "", false
}
cmdEnv := types.NewMappingWithEquals(opts.Environment)
serviceOverrideEnv := cmdEnv.Resolve(func(s string) (string, bool) {
v, ok := project.Environment[s]
return v, ok
}).RemoveEmpty()
service.Environment.OverrideBy(projectEnv)
service.Environment.OverrideBy(serviceOverrideEnv)
}
for k, v := range opts.Labels {
service.Labels = service.Labels.Add(k, v)
Expand Down
189 changes: 124 additions & 65 deletions pkg/e2e/compose_environment_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,137 +27,196 @@ import (
func TestEnvPriority(t *testing.T) {
c := NewParallelCLI(t)

projectDir := "./fixtures/environment/env-priority"

t.Run("up", func(t *testing.T) {
c.RunDockerOrExitError(t, "rmi", "env-compose-priority")
c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose-with-env.yaml",
"--project-directory", projectDir, "up", "-d", "--build")
"up", "-d", "--build")
})

// Full options activated
// 1. Compose file <-- Result expected
// 2. Shell environment variables
// 3. Environment file
// 4. Dockerfile
// 1. Command Line (docker compose run --env <KEY[=VAL]>) <-- Result expected (From environment patched by --env-file)
// 2. Compose File (service::environment section)
// 3. Compose File (service::env_file section file)
// 4. Container Image ENV directive
// 5. Variable is not defined
t.Run("compose file priority", func(t *testing.T) {
cmd := c.NewDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose-with-env.yaml",
"--project-directory", projectDir, "--env-file", "./fixtures/environment/env-priority/.env.override", "run",
"--rm", "-e", "WHEREAMI", "env-compose-priority")
"--env-file", "./fixtures/environment/env-priority/.env.override",
"run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So there's a difference between -e WHEREAMI with the var defined, and -e WHEREAMI=shell? In this case shouldn't 3. be the expected result?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the first, the variable value will be inherited from the environment.
The second one will have the explicit value passed (shell in this case).

cmd.Env = append(cmd.Env, "WHEREAMI=shell")
res := icmd.RunCmd(cmd)
assert.Equal(t, strings.TrimSpace(res.Stdout()), "Compose File")
assert.Equal(t, strings.TrimSpace(res.Stdout()), "override")
})

// Full options activated
// 1. Command Line (docker compose run --env <KEY[=VAL]>) <-- Result expected
// 2. Compose File (service::environment section)
// 3. Compose File (service::env_file section file)
// 4. Container Image ENV directive
// 5. Variable is not defined
t.Run("compose file priority", func(t *testing.T) {
cmd := c.NewDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose-with-env.yaml",
"--env-file", "./fixtures/environment/env-priority/.env.override",
"run", "--rm", "-e", "WHEREAMI=shell", "env-compose-priority")
res := icmd.RunCmd(cmd)
assert.Equal(t, strings.TrimSpace(res.Stdout()), "shell")
})

// No Compose file, all other options
// 1. Compose file
// 2. Shell environment variables <-- Result expected
// 3. Environment file
// 4. Dockerfile
// 1. Command Line (docker compose run --env <KEY[=VAL]>) <-- Result expected (From environment patched by --env-file)
// 2. Compose File (service::environment section)
// 3. Compose File (service::env_file section file)
// 4. Container Image ENV directive
// 5. Variable is not defined
t.Run("shell priority", func(t *testing.T) {
cmd := c.NewDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml", "--project-directory",
projectDir, "--env-file", "./fixtures/environment/env-priority/.env.override", "run", "--rm", "-e",
"WHEREAMI", "env-compose-priority")
cmd := c.NewDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml",
"--env-file", "./fixtures/environment/env-priority/.env.override",
"run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
cmd.Env = append(cmd.Env, "WHEREAMI=shell")
res := icmd.RunCmd(cmd)
assert.Equal(t, strings.TrimSpace(res.Stdout()), "override")
})

// No Compose file, all other options with env variable from OS environment
// 1. Command Line (docker compose run --env <KEY[=VAL]>) <-- Result expected (From environment)
// 2. Compose File (service::environment section)
// 3. Compose File (service::env_file section file)
// 4. Container Image ENV directive
// 5. Variable is not defined
t.Run("shell priority file with default value", func(t *testing.T) {
cmd := c.NewDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml",
"--env-file", "./fixtures/environment/env-priority/.env.override.with.default",
"run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
cmd.Env = append(cmd.Env, "WHEREAMI=shell")
res := icmd.RunCmd(cmd)
assert.Equal(t, strings.TrimSpace(res.Stdout()), "shell")
})

// No Compose file and env variable pass to the run command
// 1. Compose file
// 2. Shell environment variables <-- Result expected
// 3. Environment file
// 4. Dockerfile
// No Compose file, all other options with env variable from OS environment
// 1. Command Line (docker compose run --env <KEY[=VAL]>) <-- Result expected (From environment default value from file in --env-file)
// 2. Compose File (service::environment section)
// 3. Compose File (service::env_file section file)
// 4. Container Image ENV directive
// 5. Variable is not defined
t.Run("shell priority implicitly set", func(t *testing.T) {
cmd := c.NewDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml",
"--env-file", "./fixtures/environment/env-priority/.env.override.with.default",
"run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
res := icmd.RunCmd(cmd)
assert.Equal(t, strings.TrimSpace(res.Stdout()), "EnvFileDefaultValue")
})

// No Compose file and env variable pass to the run command
// 1. Command Line (docker compose run --env <KEY[=VAL]>) <-- Result expected
// 2. Compose File (service::environment section)
// 3. Compose File (service::env_file section file)
// 4. Container Image ENV directive
// 5. Variable is not defined
t.Run("shell priority from run command", func(t *testing.T) {
res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml", "--project-directory",
projectDir, "--env-file", "./fixtures/environment/env-priority/.env.override", "run", "--rm", "-e",
"WHEREAMI=shell-run", "env-compose-priority")
res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml",
"--env-file", "./fixtures/environment/env-priority/.env.override",
"run", "--rm", "-e", "WHEREAMI=shell-run", "env-compose-priority")
assert.Equal(t, strings.TrimSpace(res.Stdout()), "shell-run")
})

// No Compose file & no env variable but override env file
// 1. Compose file
// 2. Shell environment variables
// 3. Environment file <-- Result expected
// 4. Dockerfile
// No Compose file & no env variable but override env file
// 1. Command Line (docker compose run --env <KEY[=VAL]>) <-- Result expected (From environment patched by .env as a default --env-file value)
// 2. Compose File (service::environment section)
// 3. Compose File (service::env_file section file)
// 4. Container Image ENV directive
// 5. Variable is not defined
t.Run("override env file from compose", func(t *testing.T) {
res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose-with-env-file.yaml",
"run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
assert.Equal(t, strings.TrimSpace(res.Stdout()), "Env File")
})

// No Compose file & no env variable but override by default env file
// 1. Command Line (docker compose run --env <KEY[=VAL]>) <-- Result expected (From environment patched by --env-file value)
// 2. Compose File (service::environment section)
// 3. Compose File (service::env_file section file)
// 4. Container Image ENV directive
// 5. Variable is not defined
t.Run("override env file", func(t *testing.T) {
res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml", "--project-directory",
projectDir, "--env-file", "./fixtures/environment/env-priority/.env.override", "run", "--rm", "-e",
"WHEREAMI", "env-compose-priority")
res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml",
"--env-file", "./fixtures/environment/env-priority/.env.override",
"run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
assert.Equal(t, strings.TrimSpace(res.Stdout()), "override")
})

// No Compose file & no env variable but override env file
// 1. Compose file
// 2. Shell environment variables
// 3. Environment file <-- Result expected
// 4. Dockerfile
// No Compose file & no env variable but override env file
// 1. Command Line (docker compose run --env <KEY[=VAL]>) <-- Result expected (From environment patched by --env-file value)
// 2. Compose File (service::environment section)
// 3. Compose File (service::env_file section file)
// 4. Container Image ENV directive
// 5. Variable is not defined
t.Run("env file", func(t *testing.T) {
res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml", "--project-directory",
projectDir, "run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml",
"run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
assert.Equal(t, strings.TrimSpace(res.Stdout()), "Env File")
})

// No Compose file & no env variable, using an empty override env file
// 1. Compose file
// 2. Shell environment variables
// 3. Environment file
// 4. Dockerfile <-- Result expected
// No Compose file & no env variable, using an empty override env file
// 1. Command Line (docker compose run --env <KEY[=VAL]>)
// 2. Compose File (service::environment section)
// 3. Compose File (service::env_file section file)
// 4. Container Image ENV directive <-- Result expected
// 5. Variable is not defined
t.Run("use Dockerfile", func(t *testing.T) {
res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml", "--project-directory",
projectDir, "--env-file", "./fixtures/environment/env-priority/.env.empty", "run", "--rm", "-e", "WHEREAMI",
"env-compose-priority")
res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml",
"--env-file", "./fixtures/environment/env-priority/.env.empty",
"run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
assert.Equal(t, strings.TrimSpace(res.Stdout()), "Dockerfile")
})

t.Run("down", func(t *testing.T) {
c.RunDockerComposeCmd(t, "--project-directory", projectDir, "down")
c.RunDockerComposeCmd(t, "--project-name", "env-priority", "down")
})
}

func TestEnvInterpolation(t *testing.T) {
c := NewParallelCLI(t)

projectDir := "./fixtures/environment/env-interpolation"

// No variable defined in the Compose file and env variable pass to the run command
// 1. Compose file
// 2. Shell environment variables <-- Result expected
// 3. Environment file
// 4. Dockerfile
// No variable defined in the Compose file and nor env variable pass to the run command
// 1. Command Line (docker compose run --env <KEY[=VAL]>)
// 2. Compose File (service::environment section) <-- Result expected (From environment patched by .env as a default --env-file value)
// 3. Compose File (service::env_file section file)
// 4. Container Image ENV directive
// 5. Variable is not defined
t.Run("shell priority from run command", func(t *testing.T) {
cmd := c.NewDockerComposeCmd(t, "-f", "./fixtures/environment/env-interpolation/compose.yaml",
"--project-directory", projectDir, "config")
cmd := c.NewDockerComposeCmd(t, "-f", "./fixtures/environment/env-interpolation/compose.yaml", "config")
cmd.Env = append(cmd.Env, "WHEREAMI=shell")
res := icmd.RunCmd(cmd)
res.Assert(t, icmd.Expected{Out: `IMAGE: default_env:shell`})
res.Assert(t, icmd.Expected{Out: `IMAGE: default_env:EnvFile`})
})

// No variable defined in the Compose file and env variable pass to the run command
// 1. Command Line (docker compose run --env <KEY[=VAL]>)
// 2. Compose File (service::environment section) <-- Result expected (From environment patched by .env as a default --env-file value.
// This variable has a default value in case of an absent variable in the OS environment)
// 3. Compose File (service::env_file section file)
// 4. Container Image ENV directive
// 5. Variable is not defined
t.Run("shell priority from run command using default value fallback", func(t *testing.T) {
c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-interpolation-default-value/compose.yaml", "config").
Assert(t, icmd.Expected{Out: `IMAGE: default_env:EnvFileDefaultValue`})
})
}

func TestCommentsInEnvFile(t *testing.T) {
c := NewParallelCLI(t)

projectDir := "./fixtures/environment/env-file-comments"

t.Run("comments in env files", func(t *testing.T) {
c.RunDockerOrExitError(t, "rmi", "env-file-comments")

c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-file-comments/compose.yaml", "--project-directory",
projectDir, "up", "-d", "--build")
c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-file-comments/compose.yaml", "up", "-d", "--build")

res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-file-comments/compose.yaml",
"--project-directory", projectDir, "run", "--rm", "-e", "COMMENT", "-e", "NO_COMMENT", "env-file-comments")
"run", "--rm", "-e", "COMMENT", "-e", "NO_COMMENT", "env-file-comments")

res.Assert(t, icmd.Expected{Out: `COMMENT=1234`})
res.Assert(t, icmd.Expected{Out: `NO_COMMENT=1234#5`})

c.RunDockerComposeCmd(t, "--project-directory", projectDir, "down", "--rmi", "all")
c.RunDockerComposeCmd(t, "--project-name", "env-file-comments", "down", "--rmi", "all")
})
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
IMAGE=default_env:${WHEREAMI:-EnvFileDefaultValue}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
services:
env-interpolation:
image: bash
environment:
IMAGE: ${IMAGE}
command: echo "$IMAGE"
4 changes: 2 additions & 2 deletions pkg/e2e/fixtures/environment/env-interpolation/.env
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
WHEREAMI=Env File
IMAGE=default_env:${WHEREAMI}
WHEREAMI=EnvFile
IMAGE=default_env:${WHEREAMI}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
WHEREAMI=${WHEREAMI:-EnvFileDefaultValue}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
services:
env-compose-priority:
image: env-compose-priority
build:
context: .
env_file:
- .env.override