Merge branch 'main' into preview-deployment-upload-artifacts

cmd/cmd_utils.go

@@ -266,7 +266,7 @@ func executeCustomCommand(
 			}
 			if component == "" || component == "<no value>" {
 				u.LogErrorAndExit(cliConfig, fmt.Errorf("the command defines an invalid 'component_config.component: %s' in '%s'",
-					commandConfig.ComponentConfig.Component, cfg.CliConfigFileName+cfg.DefaultStackConfigFileExtension))
+					commandConfig.ComponentConfig.Component, cfg.CliConfigFileName+u.DefaultStackConfigFileExtension))
 			}
 
 			// Process Go templates in the command's 'component_config.stack'
@@ -276,7 +276,7 @@ func executeCustomCommand(
 			}
 			if stack == "" || stack == "<no value>" {
 				u.LogErrorAndExit(cliConfig, fmt.Errorf("the command defines an invalid 'component_config.stack: %s' in '%s'",
-					commandConfig.ComponentConfig.Stack, cfg.CliConfigFileName+cfg.DefaultStackConfigFileExtension))
+					commandConfig.ComponentConfig.Stack, cfg.CliConfigFileName+u.DefaultStackConfigFileExtension))
 			}
 
 			// Get the config for the component in the stack

internal/exec/stack_processor_utils.go

@@ -66,7 +66,7 @@ func ProcessYAMLConfigFiles(
 			stackFileName := strings.TrimSuffix(
 				strings.TrimSuffix(
 					u.TrimBasePathFromPath(stackBasePath+"/", p),
-					cfg.DefaultStackConfigFileExtension),
+					u.DefaultStackConfigFileExtension),
 				".yml",
 			)
 
@@ -371,8 +371,33 @@ func ProcessYAMLConfigFile(
 		impWithExt := imp
 		ext := filepath.Ext(imp)
 		if ext == "" {
-			ext = cfg.DefaultStackConfigFileExtension
-			impWithExt = imp + ext
+			extensions := []string{
+				u.YamlFileExtension,
+				u.YmlFileExtension,
+				u.YamlTemplateExtension,
+				u.YmlTemplateExtension,
+			}
+
+			found := false
+			for _, extension := range extensions {
+				testPath := path.Join(basePath, imp+extension)
+				if _, err := os.Stat(testPath); err == nil {
+					impWithExt = imp + extension
+					found = true
+					break
+				}
+			}
+
+			if !found {
+				// Default to .yaml if no file is found
+				impWithExt = imp + u.DefaultStackConfigFileExtension
+			}
+		} else if ext == u.YamlFileExtension || ext == u.YmlFileExtension {
+			// Check if there's a template version of this file
+			templatePath := impWithExt + u.TemplateExtension
+			if _, err := os.Stat(path.Join(basePath, templatePath)); err == nil {
+				impWithExt = templatePath
+			}
 		}
 
 		impWithExtPath := path.Join(basePath, impWithExt)
@@ -452,7 +477,7 @@ func ProcessYAMLConfigFile(
 			importRelativePathWithExt := strings.Replace(importFile, basePath+"/", "", 1)
 			ext2 := filepath.Ext(importRelativePathWithExt)
 			if ext2 == "" {
-				ext2 = cfg.DefaultStackConfigFileExtension
+				ext2 = u.DefaultStackConfigFileExtension
 			}
 			importRelativePathWithoutExt := strings.TrimSuffix(importRelativePathWithExt, ext2)
 			importsConfig[importRelativePathWithoutExt] = yamlConfigRaw
@@ -493,7 +518,7 @@ func ProcessStackConfig(
 	stackName := strings.TrimSuffix(
 		strings.TrimSuffix(
 			u.TrimBasePathFromPath(stacksBasePath+"/", stack),
-			cfg.DefaultStackConfigFileExtension),
+			u.DefaultStackConfigFileExtension),
 		".yml",
 	)
 
@@ -1859,13 +1884,13 @@ func CreateComponentStackMap(
 
 	for stack, components := range stackComponentMap["terraform"] {
 		for _, component := range components {
-			componentStackMap["terraform"][component] = append(componentStackMap["terraform"][component], strings.Replace(stack, cfg.DefaultStackConfigFileExtension, "", 1))
+			componentStackMap["terraform"][component] = append(componentStackMap["terraform"][component], strings.Replace(stack, u.DefaultStackConfigFileExtension, "", 1))
 		}
 	}
 
 	for stack, components := range stackComponentMap["helmfile"] {
 		for _, component := range components {
-			componentStackMap["helmfile"][component] = append(componentStackMap["helmfile"][component], strings.Replace(stack, cfg.DefaultStackConfigFileExtension, "", 1))
+			componentStackMap["helmfile"][component] = append(componentStackMap["helmfile"][component], strings.Replace(stack, u.DefaultStackConfigFileExtension, "", 1))
 		}
 	}
 

internal/exec/workflow.go

@@ -86,7 +86,7 @@ func ExecuteWorkflowCmd(cmd *cobra.Command, args []string) error {
 	// If the workflow file is specified without an extension, use the default extension
 	ext := filepath.Ext(workflowPath)
 	if ext == "" {
-		ext = cfg.DefaultStackConfigFileExtension
+		ext = u.DefaultStackConfigFileExtension
 		workflowPath = workflowPath + ext
 	}
 

pkg/config/const.go

@@ -1,11 +1,9 @@
 package config
 
 const (
-	DefaultStackConfigFileExtension       = ".yaml"
-	DefaultVendoringManifestFileExtension = ".yaml"
-	CliConfigFileName                     = "atmos"
-	SystemDirConfigFilePath               = "/usr/local/etc/atmos"
-	WindowsAppDataEnvVar                  = "LOCALAPPDATA"
+	CliConfigFileName       = "atmos"
+	SystemDirConfigFilePath = "/usr/local/etc/atmos"
+	WindowsAppDataEnvVar    = "LOCALAPPDATA"
 
 	// GlobalOptionsFlag is a custom flag to specify helmfile `GLOBAL OPTIONS`
 	// https://github.com/roboll/helmfile#cli-reference

pkg/config/utils.go

@@ -25,36 +25,46 @@ func FindAllStackConfigsInPathsForStack(
 	var stackIsDir = strings.IndexAny(stack, "/") > 0
 
 	for _, p := range includeStackPaths {
-		pathWithExt := p
+		// Try both regular and template patterns
+		patterns := []string{p}
 
 		ext := filepath.Ext(p)
 		if ext == "" {
-			ext = DefaultStackConfigFileExtension
-			pathWithExt = p + ext
+			patterns = getStackFilePatterns(p)
 		}
 
-		// Find all matches in the glob
-		matches, err := u.GetGlobMatches(pathWithExt)
-		if err != nil || len(matches) == 0 {
-			// Retry (b/c we are using `doublestar` library, and it sometimes has issues reading many files in a Docker container)
-			// TODO: review `doublestar` library
-			matches, err = u.GetGlobMatches(pathWithExt)
+		var allMatches []string
+		for _, pattern := range patterns {
+			// Find all matches in the glob
+			matches, err := u.GetGlobMatches(pattern)
+			if err == nil && len(matches) > 0 {
+				allMatches = append(allMatches, matches...)
+			}
+		}
+
+		// If no matches were found across all patterns, we perform an additional check:
+		// We try to get matches for the first pattern only to determine if there's a genuine error
+		// (like permission issues or invalid path) versus simply no matching files.
+		if len(allMatches) == 0 {
+			_, err := u.GetGlobMatches(patterns[0])
 			if err != nil {
 				if cliConfig.Logs.Level == u.LogLevelTrace {
 					y, _ := u.ConvertToYAML(cliConfig)
 					return nil, nil, false, fmt.Errorf("%v\n\n\nCLI config:\n\n%v", err, y)
 				}
 				return nil, nil, false, err
 			}
-
+			// If there's no error but still no matches, we continue to the next path
+			// This happens when the pattern is valid but no files match it
+			continue
 		}
 
-		// Exclude files that match any of the excludePaths
-		for _, matchedFileAbsolutePath := range matches {
+		// Process all matches found
+		for _, matchedFileAbsolutePath := range allMatches {
 			matchedFileRelativePath := u.TrimBasePathFromPath(cliConfig.StacksBaseAbsolutePath+"/", matchedFileAbsolutePath)
 
 			// Check if the provided stack matches a file in the config folders (excluding the files from `excludeStackPaths`)
-			stackMatch := strings.HasSuffix(matchedFileAbsolutePath, stack+DefaultStackConfigFileExtension)
+			stackMatch := matchesStackFilePattern(matchedFileAbsolutePath, stack)
 
 			if stackMatch {
 				allExcluded := true
@@ -93,6 +103,10 @@ func FindAllStackConfigsInPathsForStack(
 		}
 	}
 
+	if len(absolutePaths) == 0 {
+		return nil, nil, false, fmt.Errorf("no matches found for the provided stack '%s' in the paths %v", stack, includeStackPaths)
+	}
+
 	return absolutePaths, relativePaths, false, nil
 }
 
@@ -107,31 +121,41 @@ func FindAllStackConfigsInPaths(
 	var relativePaths []string
 
 	for _, p := range includeStackPaths {
-		pathWithExt := p
+		patterns := []string{p}
 
 		ext := filepath.Ext(p)
 		if ext == "" {
-			ext = DefaultStackConfigFileExtension
-			pathWithExt = p + ext
+			patterns = getStackFilePatterns(p)
 		}
 
-		// Find all matches in the glob
-		matches, err := u.GetGlobMatches(pathWithExt)
-		if err != nil || len(matches) == 0 {
-			// Retry (b/c we are using `doublestar` library, and it sometimes has issues reading many files in a Docker container)
-			// TODO: review `doublestar` library
-			matches, err = u.GetGlobMatches(pathWithExt)
+		var allMatches []string
+		for _, pattern := range patterns {
+			// Find all matches in the glob
+			matches, err := u.GetGlobMatches(pattern)
+			if err == nil && len(matches) > 0 {
+				allMatches = append(allMatches, matches...)
+			}
+		}
+
+		// If no matches were found across all patterns, we perform an additional check:
+		// We try to get matches for the first pattern only to determine if there's a genuine error
+		// (like permission issues or invalid path) versus simply no matching files.
+		if len(allMatches) == 0 {
+			_, err := u.GetGlobMatches(patterns[0])
 			if err != nil {
 				if cliConfig.Logs.Level == u.LogLevelTrace {
 					y, _ := u.ConvertToYAML(cliConfig)
 					return nil, nil, fmt.Errorf("%v\n\n\nCLI config:\n\n%v", err, y)
 				}
 				return nil, nil, err
 			}
+			// If there's no error but still no matches, we continue to the next path
+			// This happens when the pattern is valid but no files match it
+			continue
 		}
 
-		// Exclude files that match any of the excludePaths
-		for _, matchedFileAbsolutePath := range matches {
+		// Process all matches found
+		for _, matchedFileAbsolutePath := range allMatches {
 			matchedFileRelativePath := u.TrimBasePathFromPath(cliConfig.StacksBaseAbsolutePath+"/", matchedFileAbsolutePath)
 			include := true
 
@@ -630,3 +654,83 @@ func GetStackNameFromContextAndStackNamePattern(
 
 	return stack, nil
 }
+
+// getStackFilePatterns returns a slice of possible file patterns for a given base path
+func getStackFilePatterns(basePath string) []string {
+	return []string{
+		basePath + u.DefaultStackConfigFileExtension,
+		basePath + u.DefaultStackConfigFileExtension + u.TemplateExtension,
+		basePath + u.YamlTemplateExtension,
+		basePath + u.YmlTemplateExtension,
+	}
+}
+
+// matchesStackFilePattern checks if a file path matches any of the valid stack file patterns
+func matchesStackFilePattern(filePath, stackName string) bool {
+	patterns := getStackFilePatterns(stackName)
+	for _, pattern := range patterns {
+		if strings.HasSuffix(filePath, pattern) {
+			return true
+		}
+	}
+	return false
+}
+
+func getConfigFilePatterns(path string, forGlobMatch bool) []string {
+	if path == "" {
+		return []string{}
+	}
+	ext := filepath.Ext(path)
+	if ext != "" {
+		return []string{path}
+	}
+
+	patterns := []string{
+		path + u.DefaultStackConfigFileExtension,
+		path + u.DefaultStackConfigFileExtension + u.TemplateExtension,
+		path + u.YamlTemplateExtension,
+		path + u.YmlTemplateExtension,
+	}
+	if !forGlobMatch {
+		// For direct file search, include the exact path without extension
+		patterns = append([]string{path}, patterns...)
+	}
+
+	return patterns
+}
+
+func SearchConfigFile(configPath string, cliConfig schema.CliConfiguration) (string, error) {
+	// If path already has an extension, verify it exists
+	if ext := filepath.Ext(configPath); ext != "" {
+		if _, err := os.Stat(configPath); err == nil {
+			return configPath, nil
+		}
+		return "", fmt.Errorf("specified config file not found: %s", configPath)
+	}
+
+	dir := filepath.Dir(configPath)
+	base := filepath.Base(configPath)
+
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return "", fmt.Errorf("error reading directory %s: %v", dir, err)
+	}
+
+	// Create a map of existing files for quick lookup
+	fileMap := make(map[string]bool)
+	for _, entry := range entries {
+		if !entry.IsDir() {
+			fileMap[entry.Name()] = true
+		}
+	}
+
+	// Try all patterns in order
+	patterns := getConfigFilePatterns(base, false)
+	for _, pattern := range patterns {
+		if fileMap[pattern] {
+			return filepath.Join(dir, pattern), nil
+		}
+	}
+
+	return "", fmt.Errorf("failed to find a match for the import '%s' ('%s' + '%s')", configPath, dir, base)
+}

pkg/utils/file_extensions.go

@@ -0,0 +1,11 @@
+package utils
+
+const (
+	DefaultStackConfigFileExtension       = ".yaml"
+	DefaultVendoringManifestFileExtension = ".yaml"
+	YamlFileExtension                     = ".yaml"
+	YmlFileExtension                      = ".yml"
+	YamlTemplateExtension                 = ".yaml.tmpl"
+	YmlTemplateExtension                  = ".yml.tmpl"
+	TemplateExtension                     = ".tmpl"
+)

pkg/utils/file_utils.go

@@ -39,8 +39,13 @@ func FileOrDirExists(filename string) bool {
 
 // IsYaml checks if the file has YAML extension (does not check file schema, nor validates the file)
 func IsYaml(file string) bool {
-	yamlExtensions := []string{".yaml", ".yml"}
+	yamlExtensions := []string{YamlFileExtension, YmlFileExtension, YamlTemplateExtension, YmlTemplateExtension}
 	ext := filepath.Ext(file)
+	if ext == ".tmpl" {
+		// For .tmpl files, we check if the full extension is .yaml.tmpl or .yml.tmpl
+		baseExt := filepath.Ext(strings.TrimSuffix(file, ext))
+		ext = baseExt + ext
+	}
 	return SliceContainsString(yamlExtensions, ext)
 }
 
@@ -180,12 +185,12 @@ func IsSocket(path string) (bool, error) {
 // If the path has a file extension, it checks if the file exists.
 // If the path does not have a file extension, it checks for the existence of the file with the provided path and the possible config file extensions
 func SearchConfigFile(path string) (string, bool) {
-	// check if the provided has a file extension
+	// check if the provided path has a file extension
 	if filepath.Ext(path) != "" {
 		return path, FileExists(path)
 	}
 	// Define the possible config file extensions
-	configExtensions := []string{".yaml", ".yml"}
+	configExtensions := []string{YamlFileExtension, YmlFileExtension, YamlTemplateExtension, YmlTemplateExtension}
 	for _, ext := range configExtensions {
 		filePath := path + ext
 		if FileExists(filePath) {

website/docs/core-concepts/stacks/imports.mdx

@@ -52,6 +52,23 @@ import:
   - catalog/file3.YAML  # Expicitly load a file with a .YAML extension
 ```
 
+### Automatic Template File Detection
+
+When importing files without specifying an extension, Atmos will now automatically search for and use template versions of the files if they exist. The search order is:
+
+1. `.yaml`
+2. `.yml`
+3. `.yaml.tmpl`
+4. `.yml.tmpl`
+
+For example, if you import `catalog/file1`, Atmos will:
+1. First look for `catalog/file1.yaml` or `catalog/file1.yml`
+2. If found, check if a template version exists (`catalog/file1.yaml.tmpl` or `catalog/file1.yml.tmpl`)
+3. Use the template version if it exists, otherwise use the regular YAML file
+4. If no files are found, default to using `.yaml` extension
+
+This feature makes it easier to work with templated configurations as you don't need to explicitly specify the template file extension - Atmos will automatically use the template version when available.
+
 ## Conventions
 
 We recommend placing all baseline "imports" in the `stacks/catalog` folder, however, they can exist anywhere.