Skip to content

Commit

Permalink
introduce include to load sub-compose projects as dependencies
Browse files Browse the repository at this point in the history
Signed-off-by: Nicolas De Loof <[email protected]>
  • Loading branch information
ndeloof committed Jun 22, 2023
1 parent 532cd92 commit a175046
Show file tree
Hide file tree
Showing 10 changed files with 336 additions and 74 deletions.
62 changes: 1 addition & 61 deletions cli/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
package cli

import (
"bytes"
"io"
"os"
"path/filepath"
Expand Down Expand Up @@ -250,7 +249,7 @@ func WithDotEnv(o *ProjectOptions) error {
if err != nil {
return err
}
envMap, err := GetEnvFromFile(o.Environment, wd, o.EnvFiles)
envMap, err := dotenv.GetEnvFromFile(o.Environment, wd, o.EnvFiles)
if err != nil {
return err
}
Expand All @@ -262,65 +261,6 @@ func WithDotEnv(o *ProjectOptions) error {
return nil
}

func GetEnvFromFile(currentEnv map[string]string, workingDir string, filenames []string) (map[string]string, error) {
envMap := make(map[string]string)

dotEnvFiles := filenames
if len(dotEnvFiles) == 0 {
dotEnvFiles = append(dotEnvFiles, filepath.Join(workingDir, ".env"))
}
for _, dotEnvFile := range dotEnvFiles {
abs, err := filepath.Abs(dotEnvFile)
if err != nil {
return envMap, err
}
dotEnvFile = abs

s, err := os.Stat(dotEnvFile)
if os.IsNotExist(err) {
if len(filenames) == 0 {
return envMap, nil
}
return envMap, errors.Errorf("Couldn't find env file: %s", dotEnvFile)
}
if err != nil {
return envMap, err
}

if s.IsDir() {
if len(filenames) == 0 {
return envMap, nil
}
return envMap, errors.Errorf("%s is a directory", dotEnvFile)
}

b, err := os.ReadFile(dotEnvFile)
if os.IsNotExist(err) {
return nil, errors.Errorf("Couldn't read env file: %s", dotEnvFile)
}
if err != nil {
return envMap, err
}

env, err := dotenv.ParseWithLookup(bytes.NewReader(b), func(k string) (string, bool) {
v, ok := currentEnv[k]
if ok {
return v, true
}
v, ok = envMap[k]
return v, ok
})
if err != nil {
return envMap, errors.Wrapf(err, "failed to read %s", dotEnvFile)
}
for k, v := range env {
envMap[k] = v
}
}

return envMap, nil
}

// WithInterpolation set ProjectOptions to enable/skip interpolation
func WithInterpolation(interpolation bool) ProjectOptionsFn {
return func(o *ProjectOptions) error {
Expand Down
5 changes: 3 additions & 2 deletions cli/options_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"strings"
"testing"

"github.com/compose-spec/compose-go/dotenv"
"gotest.tools/v3/assert"

"github.com/compose-spec/compose-go/consts"
Expand Down Expand Up @@ -316,10 +317,10 @@ func TestGetEnvFromFile(t *testing.T) {
err := os.Mkdir(f, 0o700)
assert.NilError(t, err)

_, err = GetEnvFromFile(nil, wd, nil)
_, err = dotenv.GetEnvFromFile(nil, wd, nil)
assert.NilError(t, err)

_, err = GetEnvFromFile(nil, wd, []string{f})
_, err = dotenv.GetEnvFromFile(nil, wd, []string{f})
assert.Check(t, strings.HasSuffix(err.Error(), ".env is a directory"))
}

Expand Down
84 changes: 84 additions & 0 deletions dotenv/env.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
/*
Copyright 2020 The Compose Specification Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package dotenv

import (
"bytes"
"os"
"path/filepath"

"github.com/pkg/errors"
)

func GetEnvFromFile(currentEnv map[string]string, workingDir string, filenames []string) (map[string]string, error) {
envMap := make(map[string]string)

dotEnvFiles := filenames
if len(dotEnvFiles) == 0 {
dotEnvFiles = append(dotEnvFiles, filepath.Join(workingDir, ".env"))
}
for _, dotEnvFile := range dotEnvFiles {
abs, err := filepath.Abs(dotEnvFile)
if err != nil {
return envMap, err
}
dotEnvFile = abs

s, err := os.Stat(dotEnvFile)
if os.IsNotExist(err) {
if len(filenames) == 0 {
return envMap, nil
}
return envMap, errors.Errorf("Couldn't find env file: %s", dotEnvFile)
}
if err != nil {
return envMap, err
}

if s.IsDir() {
if len(filenames) == 0 {
return envMap, nil
}
return envMap, errors.Errorf("%s is a directory", dotEnvFile)
}

b, err := os.ReadFile(dotEnvFile)
if os.IsNotExist(err) {
return nil, errors.Errorf("Couldn't read env file: %s", dotEnvFile)
}
if err != nil {
return envMap, err
}

env, err := ParseWithLookup(bytes.NewReader(b), func(k string) (string, bool) {
v, ok := currentEnv[k]
if ok {
return v, true
}
v, ok = envMap[k]
return v, ok
})
if err != nil {
return envMap, errors.Wrapf(err, "failed to read %s", dotEnvFile)
}
for k, v := range env {
envMap[k] = v
}
}

return envMap, nil
}
124 changes: 124 additions & 0 deletions loader/include.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
/*
Copyright 2020 The Compose Specification Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package loader

import (
"fmt"
"path/filepath"

"github.com/compose-spec/compose-go/dotenv"
"github.com/compose-spec/compose-go/types"
"github.com/pkg/errors"
)

// LoadIncludeConfig parse the require config from raw yaml
func LoadIncludeConfig(source []interface{}) ([]types.IncludeConfig, error) {
var requires []types.IncludeConfig
err := Transform(source, &requires)
return requires, err
}

var transformRequireConfig TransformerFunc = func(data interface{}) (interface{}, error) {
switch value := data.(type) {
case string:
return map[string]interface{}{"path": value}, nil
case map[string]interface{}:
return value, nil
default:
return data, errors.Errorf("invalid type %T for `require` configuration", value)
}
}

func loadInclude(configDetails types.ConfigDetails, model *types.Config, options []func(*Options)) (*types.Config, error) {
for _, r := range model.Include {
for i, p := range r.Path {
if !filepath.IsAbs(p) {
r.Path[i] = filepath.Join(configDetails.WorkingDir, p)
}
}
if r.ProjectDirectory == "" {
r.ProjectDirectory = filepath.Dir(r.Path[0])
}

loadOptions := []func(*Options){
func(options *Options) {
options.SetProjectName(model.Name, true)
options.ResolvePaths = true
options.SkipNormalization = true
options.SkipConsistencyCheck = true
},
}
loadOptions = append(loadOptions, options...)

env, err := dotenv.GetEnvFromFile(configDetails.Environment, r.ProjectDirectory, r.EnvFile)
if err != nil {
return nil, err
}

imported, err := Load(types.ConfigDetails{
WorkingDir: r.ProjectDirectory,
ConfigFiles: types.ToConfigFiles(r.Path),
Environment: env,
}, loadOptions...)
if err != nil {
return nil, err
}

err = importResources(model, imported, r.Path)
if err != nil {
return nil, err
}
}
model.Include = nil
return model, nil
}

// importResources import into model all resources defined by imported, and report error on conflict
func importResources(model *types.Config, imported *types.Project, path []string) error {
services := mapByName(model.Services)
for _, service := range imported.Services {
if _, ok := services[service.Name]; ok {
return fmt.Errorf("imported compose file %s defines conflicting service %s", path, service.Name)
}
model.Services = append(model.Services, service)
}
for n, network := range imported.Networks {
if _, ok := model.Networks[n]; ok {
return fmt.Errorf("imported compose file %s defines conflicting network %s", path, n)
}
model.Networks[n] = network
}
for n, volume := range imported.Volumes {
if _, ok := model.Volumes[n]; ok {
return fmt.Errorf("imported compose file %s defines conflicting volume %s", path, n)
}
model.Volumes[n] = volume
}
for n, secret := range imported.Secrets {
if _, ok := model.Secrets[n]; ok {
return fmt.Errorf("imported compose file %s defines conflicting secret %s", path, n)
}
model.Secrets[n] = secret
}
for n, config := range imported.Configs {
if _, ok := model.Configs[n]; ok {
return fmt.Errorf("imported compose file %s defines conflicting config %s", path, n)
}
model.Configs[n] = config
}
return nil
}
24 changes: 23 additions & 1 deletion loader/loader.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ type Options struct {
SkipConsistencyCheck bool
// Skip extends
SkipExtends bool
// SkipInclude will ignore `include` and only load model from file(s) set by ConfigDetails
SkipInclude bool
// Interpolation options
Interpolate *interp.Options
// Discard 'env_file' entries after resolving to 'environment' section
Expand Down Expand Up @@ -232,10 +234,18 @@ func Load(configDetails types.ConfigDetails, options ...func(*Options)) (*types.
return nil, err
}

if !opts.SkipInclude {
cfg, err = loadInclude(configDetails, cfg, options)
if err != nil {
return nil, err
}
}

if i == 0 {
model = cfg
continue
}

merged, err := merge([]*types.Config{model, cfg})
if err != nil {
return nil, err
Expand Down Expand Up @@ -428,7 +438,6 @@ func loadSections(filename string, config map[string]interface{}, configDetails
if err != nil {
return nil, err
}

cfg.Networks, err = LoadNetworks(getSection(config, "networks"))
if err != nil {
return nil, err
Expand All @@ -445,6 +454,10 @@ func loadSections(filename string, config map[string]interface{}, configDetails
if err != nil {
return nil, err
}
cfg.Include, err = LoadIncludeConfig(getSequence(config, "include"))
if err != nil {
return nil, err
}
extensions := getSection(config, extensions)
if len(extensions) > 0 {
cfg.Extensions = extensions
Expand All @@ -460,6 +473,14 @@ func getSection(config map[string]interface{}, key string) map[string]interface{
return section.(map[string]interface{})
}

func getSequence(config map[string]interface{}, key string) []interface{} {
section, ok := config[key]
if !ok {
return make([]interface{}, 0)
}
return section.([]interface{})
}

// ForbiddenPropertiesError is returned when there are properties in the Compose
// file that are forbidden.
type ForbiddenPropertiesError struct {
Expand Down Expand Up @@ -524,6 +545,7 @@ func createTransformHook(additionalTransformers ...Transformer) mapstructure.Dec
reflect.TypeOf(types.ExtendsConfig{}): transformExtendsConfig,
reflect.TypeOf(types.DeviceRequest{}): transformServiceDeviceRequest,
reflect.TypeOf(types.SSHConfig{}): transformSSHConfig,
reflect.TypeOf(types.IncludeConfig{}): transformRequireConfig,
}

for _, transformer := range additionalTransformers {
Expand Down
Loading

0 comments on commit a175046

Please sign in to comment.