process extends recursively

Signed-off-by: Nicolas De Loof <[email protected]>

Makefile

@@ -15,8 +15,8 @@
 IMAGE_PREFIX=composespec/conformance-tests-
 
 .PHONY: build
-build: ## Run tests
-	go build ./...
+build: ## Build command line
+	go build -o compose-spec cmd/main.go
 
 .PHONY: test
 test: ## Run tests

loader/extends.go

@@ -35,96 +35,122 @@ func ApplyExtends(ctx context.Context, dict map[string]any, opts *Options, track
 	if !ok {
 		return fmt.Errorf("services must be a mapping")
 	}
-	for name, s := range services {
-		service, ok := s.(map[string]any)
-		if !ok {
-			return fmt.Errorf("services.%s must be a mapping", name)
-		}
-		x, ok := service["extends"]
-		if !ok {
-			continue
-		}
-		ct, err := tracker.Add(ctx.Value(consts.ComposeFileKey{}).(string), name)
+	for name := range services {
+		merged, err := applyServiceExtends(ctx, name, services, opts, tracker, post...)
 		if err != nil {
 			return err
 		}
-		var (
-			ref  string
-			file any
-		)
-		switch v := x.(type) {
-		case map[string]any:
-			ref = v["service"].(string)
-			file = v["file"]
-		case string:
-			ref = v
+		services[name] = merged
+	}
+	dict["services"] = services
+	return nil
+}
+
+func applyServiceExtends(ctx context.Context, name string, services map[string]any, opts *Options, tracker *cycleTracker, post ...PostProcessor) (any, error) {
+	s := services[name]
+	service, ok := s.(map[string]any)
+	if !ok {
+		return nil, fmt.Errorf("services.%s must be a mapping", name)
+	}
+	extends, ok := service["extends"]
+	if !ok {
+		return s, nil
+	}
+	filename := ctx.Value(consts.ComposeFileKey{}).(string)
+	tracker, err := tracker.Add(filename, name)
+	if err != nil {
+		return nil, err
+	}
+	var (
+		ref  string
+		file any
+	)
+	switch v := extends.(type) {
+	case map[string]any:
+		ref = v["service"].(string)
+		file = v["file"]
+	case string:
+		ref = v
+	}
+
+	var base any
+	if file != nil {
+		path := file.(string)
+		allServicesFromFile, target, err := getExtendsBaseFromFile(ctx, ref, path, opts, tracker)
+		if err != nil {
+			return nil, err
+		}
+		services, base = allServicesFromFile, target
+	} else {
+		target, ok := services[ref]
+		if !ok {
+			return nil, fmt.Errorf("cannot extend service %q in %s: service not found", name, filename)
 		}
+		base = target
+	}
+	// recursively apply `extends`
+	base, err = applyServiceExtends(ctx, ref, services, opts, tracker, post...)
+	if err != nil {
+		return nil, err
+	}
 
-		var base any
-		if file != nil {
-			path := file.(string)
-			for _, loader := range opts.ResourceLoaders {
-				if !loader.Accept(path) {
-					continue
-				}
-				local, err := loader.Load(ctx, path)
-				if err != nil {
-					return err
-				}
-				localdir := filepath.Dir(local)
-				relworkingdir := loader.Dir(path)
+	source := deepClone(base).(map[string]any)
+	for _, processor := range post {
+		processor.Apply(map[string]any{
+			"services": map[string]any{
+				name: source,
+			},
+		})
+	}
+	merged, err := override.ExtendService(source, service)
+	if err != nil {
+		return nil, err
+	}
+	delete(merged, "extends")
+	return merged, nil
+}
 
-				extendsOpts := opts.clone()
-				extendsOpts.ResourceLoaders = append([]ResourceLoader{}, opts.ResourceLoaders...)
-				// replace localResourceLoader with a new flavour, using extended file base path
-				extendsOpts.ResourceLoaders[len(opts.ResourceLoaders)-1] = localResourceLoader{
-					WorkingDir: localdir,
-				}
-				extendsOpts.ResolvePaths = true
-				extendsOpts.SkipNormalization = true
-				extendsOpts.SkipConsistencyCheck = true
-				extendsOpts.SkipInclude = true
-				source, err := loadYamlModel(ctx, types.ConfigDetails{
-					WorkingDir: relworkingdir,
-					ConfigFiles: []types.ConfigFile{
-						{Filename: local},
-					},
-				}, extendsOpts, ct, nil)
-				if err != nil {
-					return err
-				}
-				services := source["services"].(map[string]any)
-				base, ok = services[ref]
-				if !ok {
-					return fmt.Errorf("cannot extend service %q in %s: service not found", name, path)
-				}
-			}
-			if base == nil {
-				return fmt.Errorf("cannot read %s", path)
-			}
-		} else {
-			base, ok = services[ref]
-			if !ok {
-				return fmt.Errorf("cannot extend service %q in %s: service not found", name, "filename") // TODO track filename
-			}
+func getExtendsBaseFromFile(ctx context.Context, name string, path string, opts *Options, ct *cycleTracker) (map[string]any, any, error) {
+	for _, loader := range opts.ResourceLoaders {
+		if !loader.Accept(path) {
+			continue
 		}
-		source := deepClone(base).(map[string]any)
-		for _, processor := range post {
-			processor.Apply(map[string]any{
-				"services": map[string]any{
-					name: source,
-				},
-			})
+		local, err := loader.Load(ctx, path)
+		if err != nil {
+			return nil, nil, err
+		}
+		localdir := filepath.Dir(local)
+		relworkingdir := loader.Dir(path)
+
+		extendsOpts := opts.clone()
+		extendsOpts.ResourceLoaders = append([]ResourceLoader{}, opts.ResourceLoaders...)
+		// replace localResourceLoader with a new flavour, using extended file base path
+		extendsOpts.ResourceLoaders[len(opts.ResourceLoaders)-1] = localResourceLoader{
+			WorkingDir: localdir,
 		}
-		merged, err := override.ExtendService(source, service)
+		extendsOpts.ResolvePaths = true
+		extendsOpts.SkipNormalization = true
+		extendsOpts.SkipConsistencyCheck = true
+		extendsOpts.SkipInclude = true
+		extendsOpts.SkipExtends = true    // we manage extends recursively based on raw service definition
+		extendsOpts.SkipValidation = true // we validate the merge result
+		source, err := loadYamlModel(ctx, types.ConfigDetails{
+			WorkingDir: relworkingdir,
+			ConfigFiles: []types.ConfigFile{
+				{Filename: local},
+			},
+		}, extendsOpts, ct, nil)
 		if err != nil {
-			return err
+			return nil, nil, err
 		}
-		delete(merged, "extends")
-		services[name] = merged
+		services := source["services"].(map[string]any)
+		base, ok := services[name]
+		if !ok {
+			return nil, nil, fmt.Errorf("cannot extend service %q in %s: service not found", name, path)
+		}
+		return services, base, nil
 	}
-	dict["services"] = services
-	return nil
+	return nil, nil, fmt.Errorf("cannot read %s", path)
 }
 
 func deepClone(value any) any {

paths/extends.go

@@ -0,0 +1,30 @@
+/*
+   Copyright 2020 The Compose Specification Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package paths
+
+import "strings"
+
+func (r *relativePathsResolver) absExtendsPath(value any) (any, error) {
+	v := value.(string)
+	i := strings.Index(v, ":")
+	if i >= 2 {
+		// path is `xx:foo/bar` which denotes a remote resource
+		// FIXME we need a more deterministic way to detect a remote resource. Maybe require a more explicit syntax?
+		return v, nil
+	}
+	return r.absPath(v)
+}

paths/resolve.go

@@ -29,12 +29,14 @@ type resolver func(any) (any, error)
 
 // ResolveRelativePaths make relative paths absolute
 func ResolveRelativePaths(project map[string]any, base string) error {
-	r := relativePathsResolver{workingDir: base}
+	r := relativePathsResolver{
+		workingDir: base,
+	}
 	r.resolvers = map[tree.Path]resolver{
 		"services.*.build.context":               r.absContextPath,
 		"services.*.build.additional_contexts.*": r.absContextPath,
 		"services.*.env_file.*.path":             r.absPath,
-		"services.*.extends.file":                r.absPath,
+		"services.*.extends.file":                r.absExtendsPath,
 		"services.*.develop.watch.*.path":        r.absPath,
 		"services.*.volumes.*":                   r.absVolumeMount,
 		"configs.*.file":                         r.maybeUnixPath,