Skip to content

Commit

Permalink
introduce ResolveRelativePaths as a distinct operation vs Normalization
Browse files Browse the repository at this point in the history 
Signed-off-by: Nicolas De Loof <[email protected]>
  • Loading branch information
@ndeloof
ndeloof committed Jun 5, 2023
1 parent dc7471e commit 0b97a90
Show file tree
Hide file tree
Showing 7 changed files with 317 additions and 256 deletions.
16 changes: 10 additions & 6 deletions loader/full-struct_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,8 +177,8 @@ func services(workingDir, homeDir string) []types.ServiceConfig {
"ENV_WITH_UNDERSCORE": strPtr("ok"),
},
EnvFile: []string{
"./example1.env",
"./example2.env",
filepath.Join(workingDir, "example1.env"),
filepath.Join(workingDir, "example2.env"),
},
Expose: []string{"3000", "8000"},
ExternalLinks: []string{
Expand Down Expand Up @@ -728,8 +728,8 @@ services:
FOO: foo_from_env_file
QUX: qux_from_environment
env_file:
- ./example1.env
- ./example2.env
- %s
- %s
expose:
- "3000"
- "8000"
Expand Down Expand Up @@ -1035,6 +1035,8 @@ x-nested:
bar: baz
foo: bar
`,
filepath.Join(workingDir, "example1.env"),
filepath.Join(workingDir, "example2.env"),
workingDir,
filepath.Join(workingDir, "static"),
filepath.Join(homeDir, "configs"),
Expand Down Expand Up @@ -1327,8 +1329,8 @@ func fullExampleJSON(workingDir, homeDir string) string {
"QUX": "qux_from_environment"
},
"env_file": [
"./example1.env",
"./example2.env"
"%s",
"%s"
],
"expose": [
"3000",
Expand Down Expand Up @@ -1677,6 +1679,8 @@ func fullExampleJSON(workingDir, homeDir string) string {
toPath(workingDir, "config_data"),
toPath(homeDir, "config_data"),
toPath(workingDir, "secret_data"),
toPath(workingDir, "example1.env"),
toPath(workingDir, "example2.env"),
toPath(workingDir),
toPath(workingDir, "static"),
toPath(homeDir, "configs"),
Expand Down
118 changes: 35 additions & 83 deletions loader/loader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,7 @@ func Load(configDetails types.ConfigDetails, options ...func(*Options)) (*types.
LookupValue: configDetails.LookupEnv,
TypeCastMapping: interpolateTypeCastMapping,
},
ResolvePaths: true,
}

for _, op := range options {
Expand Down Expand Up @@ -248,14 +249,6 @@ func Load(configDetails types.ConfigDetails, options ...func(*Options)) (*types.
model = merged
}

for _, s := range model.Services {
var newEnvFiles types.StringList
for _, ef := range s.EnvFile {
newEnvFiles = append(newEnvFiles, absPath(configDetails.WorkingDir, ef))
}
s.EnvFile = newEnvFiles
}

project := &types.Project{
Name: projectName,
WorkingDir: configDetails.WorkingDir,
Expand All @@ -268,8 +261,24 @@ func Load(configDetails types.ConfigDetails, options ...func(*Options)) (*types.
Extensions: model.Extensions,
}

if opts.ResolvePaths {
err = ResolveRelativePaths(project)
if err != nil {
return nil, err
}
}

if opts.ConvertWindowsPaths {
for i, service := range project.Services {
for j, volume := range service.Volumes {
service.Volumes[j] = convertVolumePath(volume)
}
project.Services[i] = service
}
}

if !opts.SkipNormalization {
err = Normalize(project, opts.ResolvePaths)
err = Normalize(project)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -428,11 +437,11 @@ func loadSections(filename string, config map[string]interface{}, configDetails
if err != nil {
return nil, err
}
cfg.Secrets, err = LoadSecrets(getSection(config, "secrets"), configDetails, opts.ResolvePaths)
cfg.Secrets, err = LoadSecrets(getSection(config, "secrets"))
if err != nil {
return nil, err
}
cfg.Configs, err = LoadConfigObjs(getSection(config, "configs"), configDetails, opts.ResolvePaths)
cfg.Configs, err = LoadConfigObjs(getSection(config, "configs"))
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -634,7 +643,7 @@ func loadServiceWithExtends(filename, name string, servicesDict map[string]inter
target = map[string]interface{}{}
}

serviceConfig, err := LoadService(name, target.(map[string]interface{}), workingDir, lookupEnv, opts.ResolvePaths, opts.ConvertWindowsPaths)
serviceConfig, err := LoadService(name, target.(map[string]interface{}))
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -672,21 +681,9 @@ func loadServiceWithExtends(filename, name string, servicesDict map[string]inter
// make the paths relative to `file` rather than `baseFilePath` so
// that the resulting paths won't be absolute if `file` isn't an
// absolute path.
baseFileParent := filepath.Dir(file)
if baseService.Build != nil {
baseService.Build.Context = resolveBuildContextPath(baseFileParent, baseService.Build.Context)
}

for i, vol := range baseService.Volumes {
if vol.Type != types.VolumeTypeBind {
continue
}
baseService.Volumes[i].Source = resolveMaybeUnixPath(vol.Source, baseFileParent, lookupEnv)
}

for i, envFile := range baseService.EnvFile {
baseService.EnvFile[i] = resolveMaybeUnixPath(envFile, baseFileParent, lookupEnv)
}
baseFileParent := filepath.Dir(file)
ResolveServiceRelativePaths(baseFileParent, baseService)
}

serviceConfig, err = _merge(baseService, serviceConfig)
Expand All @@ -699,22 +696,9 @@ func loadServiceWithExtends(filename, name string, servicesDict map[string]inter
return serviceConfig, nil
}

func resolveBuildContextPath(baseFileParent string, context string) string {
// Checks if the context is an HTTP(S) URL or a remote git repository URL
for _, prefix := range []string{"https://", "http://", "git://", "github.com/", "git@"} {
if strings.HasPrefix(context, prefix) {
return context
}
}

// Note that the Dockerfile is always defined relative to the
// build context, so there's no need to update the Dockerfile field.
return absPath(baseFileParent, context)
}

// LoadService produces a single ServiceConfig from a compose file Dict
// the serviceDict is not validated if directly used. Use Load() to enable validation
func LoadService(name string, serviceDict map[string]interface{}, workingDir string, lookupEnv template.Mapping, resolvePaths bool, convertPaths bool) (*types.ServiceConfig, error) {
func LoadService(name string, serviceDict map[string]interface{}) (*types.ServiceConfig, error) {
serviceConfig := &types.ServiceConfig{
Scale: 1,
}
Expand All @@ -731,13 +715,6 @@ func LoadService(name string, serviceDict map[string]interface{}, workingDir str
return nil, errors.New(`invalid mount config for type "bind": field Source must not be empty`)
}

if resolvePaths || convertPaths {
volume = resolveVolumePath(volume, workingDir, lookupEnv)
}

if convertPaths {
volume = convertVolumePath(volume)
}
serviceConfig.Volumes[i] = volume
}

Expand All @@ -759,8 +736,8 @@ func convertVolumePath(volume types.ServiceVolumeConfig) types.ServiceVolumeConf
return volume
}

func resolveMaybeUnixPath(path string, workingDir string, lookupEnv template.Mapping) string {
filePath := expandUser(path, lookupEnv)
func resolveMaybeUnixPath(workingDir string, path string) string {
filePath := expandUser(path)
// Check if source is an absolute path (either Unix or Windows), to
// handle a Windows client with a Unix daemon or vice-versa.
//
Expand All @@ -773,20 +750,15 @@ func resolveMaybeUnixPath(path string, workingDir string, lookupEnv template.Map
return filePath
}

func resolveVolumePath(volume types.ServiceVolumeConfig, workingDir string, lookupEnv template.Mapping) types.ServiceVolumeConfig {
volume.Source = resolveMaybeUnixPath(volume.Source, workingDir, lookupEnv)
return volume
}

func resolveSecretsPath(secret types.SecretConfig, workingDir string, lookupEnv template.Mapping) types.SecretConfig {
if !secret.External.External && secret.File != "" {
secret.File = resolveMaybeUnixPath(secret.File, workingDir, lookupEnv)
secret.File = resolveMaybeUnixPath(workingDir, secret.File)
}
return secret
}

// TODO: make this more robust
func expandUser(path string, lookupEnv template.Mapping) string {
func expandUser(path string) string {
if strings.HasPrefix(path, "~") {
home, err := os.UserHomeDir()
if err != nil {
Expand Down Expand Up @@ -886,44 +858,39 @@ func LoadVolumes(source map[string]interface{}) (map[string]types.VolumeConfig,

// LoadSecrets produces a SecretConfig map from a compose file Dict
// the source Dict is not validated if directly used. Use Load() to enable validation
func LoadSecrets(source map[string]interface{}, details types.ConfigDetails, resolvePaths bool) (map[string]types.SecretConfig, error) {
func LoadSecrets(source map[string]interface{}) (map[string]types.SecretConfig, error) {
secrets := make(map[string]types.SecretConfig)
if err := Transform(source, &secrets); err != nil {
return secrets, err
}
for name, secret := range secrets {
obj, err := loadFileObjectConfig(name, "secret", types.FileObjectConfig(secret), details, false)
obj, err := loadFileObjectConfig(name, "secret", types.FileObjectConfig(secret))
if err != nil {
return nil, err
}
secretConfig := types.SecretConfig(obj)
if resolvePaths {
secretConfig = resolveSecretsPath(secretConfig, details.WorkingDir, details.LookupEnv)
}
secrets[name] = secretConfig
secrets[name] = types.SecretConfig(obj)
}
return secrets, nil
}

// LoadConfigObjs produces a ConfigObjConfig map from a compose file Dict
// the source Dict is not validated if directly used. Use Load() to enable validation
func LoadConfigObjs(source map[string]interface{}, details types.ConfigDetails, resolvePaths bool) (map[string]types.ConfigObjConfig, error) {
func LoadConfigObjs(source map[string]interface{}) (map[string]types.ConfigObjConfig, error) {
configs := make(map[string]types.ConfigObjConfig)
if err := Transform(source, &configs); err != nil {
return configs, err
}
for name, config := range configs {
obj, err := loadFileObjectConfig(name, "config", types.FileObjectConfig(config), details, resolvePaths)
obj, err := loadFileObjectConfig(name, "config", types.FileObjectConfig(config))
if err != nil {
return nil, err
}
configConfig := types.ConfigObjConfig(obj)
configs[name] = configConfig
configs[name] = types.ConfigObjConfig(obj)
}
return configs, nil
}

func loadFileObjectConfig(name string, objType string, obj types.FileObjectConfig, details types.ConfigDetails, resolvePaths bool) (types.FileObjectConfig, error) {
func loadFileObjectConfig(name string, objType string, obj types.FileObjectConfig) (types.FileObjectConfig, error) {
// if "external: true"
switch {
case obj.External.External:
Expand All @@ -943,26 +910,11 @@ func loadFileObjectConfig(name string, objType string, obj types.FileObjectConfi
if obj.File != "" {
return obj, errors.Errorf("%[1]s %[2]s: %[1]s.driver and %[1]s.file conflict; only use %[1]s.driver", objType, name)
}
default:
if obj.File != "" && resolvePaths {
obj.File = absPath(details.WorkingDir, obj.File)
}
}

return obj, nil
}

func absPath(workingDir string, filePath string) string {
if strings.HasPrefix(filePath, "~") {
home, _ := os.UserHomeDir()
return filepath.Join(home, filePath[1:])
}
if filepath.IsAbs(filePath) {
return filePath
}
return filepath.Join(workingDir, filePath)
}

var transformMapStringString TransformerFunc = func(data interface{}) (interface{}, error) {
switch value := data.(type) {
case map[string]interface{}:
Expand Down
9 changes: 4 additions & 5 deletions loader/loader_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -843,6 +843,7 @@ services:
// Default behavior keeps the `env_file` entries
configWithEnvFiles, err := Load(configDetails, func(options *Options) {
options.SkipNormalization = true
options.ResolvePaths = false
})
assert.NilError(t, err)
assert.DeepEqual(t, configWithEnvFiles.Services[0].EnvFile, types.StringList{"example1.env",
Expand Down Expand Up @@ -1453,8 +1454,7 @@ func TestLoadSecretsWarnOnDeprecatedExternalNameVersion35(t *testing.T) {
},
},
}
details := types.ConfigDetails{}
secrets, err := LoadSecrets(source, details, true)
secrets, err := LoadSecrets(source)
assert.NilError(t, err)
expected := map[string]types.SecretConfig{
"foo": {
Expand Down Expand Up @@ -1929,8 +1929,7 @@ func TestLoadWithExtends(t *testing.T) {
actual, err := Load(configDetails)
assert.NilError(t, err)

extendsDir, err := filepath.Abs(filepath.Join("testdata", "subdir"))
assert.NilError(t, err)
extendsDir := filepath.Join("testdata", "subdir")

expectedEnvFilePath := filepath.Join(extendsDir, "extra.env")

Expand Down Expand Up @@ -2089,7 +2088,7 @@ func TestLoadServiceWithVolumes(t *testing.T) {
},
},
}
s, err := LoadService("Test Name", m, ".", nil, true, false)
s, err := LoadService("Test Name", m)
assert.NilError(t, err)
assert.Equal(t, len(s.Volumes), 2)
assert.Equal(t, "/path 1", s.Volumes[0].Target)
Expand Down
Loading

0 comments on commit 0b97a90

Please sign in to comment.