From 822ab1739ef4fcda1f0002353257e326d15f5a4f Mon Sep 17 00:00:00 2001 From: "Vinicius C." Date: Fri, 6 Dec 2024 00:46:34 +0000 Subject: [PATCH] Added Support for Automatic Templated File Search for Atmos Imports (#795) * added support auto detect tmpl files * clean up * path check * clean code variables extensions * clean code variables extensions * clean code variables extensions * remove hardcode extensions * utils code clean up * docs for auto templates * extension use const * clean up --------- Co-authored-by: Andriy Knysh --- cmd/cmd_utils.go | 4 +- internal/exec/stack_processor_utils.go | 39 ++++- internal/exec/workflow.go | 2 +- pkg/config/const.go | 8 +- pkg/config/utils.go | 152 +++++++++++++++--- pkg/utils/file_extensions.go | 11 ++ pkg/utils/file_utils.go | 11 +- website/docs/core-concepts/stacks/imports.mdx | 17 ++ 8 files changed, 202 insertions(+), 42 deletions(-) create mode 100644 pkg/utils/file_extensions.go diff --git a/cmd/cmd_utils.go b/cmd/cmd_utils.go index 77858249c..c3c52e3f3 100644 --- a/cmd/cmd_utils.go +++ b/cmd/cmd_utils.go @@ -266,7 +266,7 @@ func executeCustomCommand( } if component == "" || component == "" { u.LogErrorAndExit(cliConfig, fmt.Errorf("the command defines an invalid 'component_config.component: %s' in '%s'", - commandConfig.ComponentConfig.Component, cfg.CliConfigFileName+cfg.DefaultStackConfigFileExtension)) + commandConfig.ComponentConfig.Component, cfg.CliConfigFileName+u.DefaultStackConfigFileExtension)) } // Process Go templates in the command's 'component_config.stack' @@ -276,7 +276,7 @@ func executeCustomCommand( } if stack == "" || stack == "" { u.LogErrorAndExit(cliConfig, fmt.Errorf("the command defines an invalid 'component_config.stack: %s' in '%s'", - commandConfig.ComponentConfig.Stack, cfg.CliConfigFileName+cfg.DefaultStackConfigFileExtension)) + commandConfig.ComponentConfig.Stack, cfg.CliConfigFileName+u.DefaultStackConfigFileExtension)) } // Get the config for the component in the stack diff --git a/internal/exec/stack_processor_utils.go b/internal/exec/stack_processor_utils.go index b616e2348..d4ba3e83d 100644 --- a/internal/exec/stack_processor_utils.go +++ b/internal/exec/stack_processor_utils.go @@ -66,7 +66,7 @@ func ProcessYAMLConfigFiles( stackFileName := strings.TrimSuffix( strings.TrimSuffix( u.TrimBasePathFromPath(stackBasePath+"/", p), - cfg.DefaultStackConfigFileExtension), + u.DefaultStackConfigFileExtension), ".yml", ) @@ -371,8 +371,33 @@ func ProcessYAMLConfigFile( impWithExt := imp ext := filepath.Ext(imp) if ext == "" { - ext = cfg.DefaultStackConfigFileExtension - impWithExt = imp + ext + extensions := []string{ + u.YamlFileExtension, + u.YmlFileExtension, + u.YamlTemplateExtension, + u.YmlTemplateExtension, + } + + found := false + for _, extension := range extensions { + testPath := path.Join(basePath, imp+extension) + if _, err := os.Stat(testPath); err == nil { + impWithExt = imp + extension + found = true + break + } + } + + if !found { + // Default to .yaml if no file is found + impWithExt = imp + u.DefaultStackConfigFileExtension + } + } else if ext == u.YamlFileExtension || ext == u.YmlFileExtension { + // Check if there's a template version of this file + templatePath := impWithExt + u.TemplateExtension + if _, err := os.Stat(path.Join(basePath, templatePath)); err == nil { + impWithExt = templatePath + } } impWithExtPath := path.Join(basePath, impWithExt) @@ -452,7 +477,7 @@ func ProcessYAMLConfigFile( importRelativePathWithExt := strings.Replace(importFile, basePath+"/", "", 1) ext2 := filepath.Ext(importRelativePathWithExt) if ext2 == "" { - ext2 = cfg.DefaultStackConfigFileExtension + ext2 = u.DefaultStackConfigFileExtension } importRelativePathWithoutExt := strings.TrimSuffix(importRelativePathWithExt, ext2) importsConfig[importRelativePathWithoutExt] = yamlConfigRaw @@ -493,7 +518,7 @@ func ProcessStackConfig( stackName := strings.TrimSuffix( strings.TrimSuffix( u.TrimBasePathFromPath(stacksBasePath+"/", stack), - cfg.DefaultStackConfigFileExtension), + u.DefaultStackConfigFileExtension), ".yml", ) @@ -1859,13 +1884,13 @@ func CreateComponentStackMap( for stack, components := range stackComponentMap["terraform"] { for _, component := range components { - componentStackMap["terraform"][component] = append(componentStackMap["terraform"][component], strings.Replace(stack, cfg.DefaultStackConfigFileExtension, "", 1)) + componentStackMap["terraform"][component] = append(componentStackMap["terraform"][component], strings.Replace(stack, u.DefaultStackConfigFileExtension, "", 1)) } } for stack, components := range stackComponentMap["helmfile"] { for _, component := range components { - componentStackMap["helmfile"][component] = append(componentStackMap["helmfile"][component], strings.Replace(stack, cfg.DefaultStackConfigFileExtension, "", 1)) + componentStackMap["helmfile"][component] = append(componentStackMap["helmfile"][component], strings.Replace(stack, u.DefaultStackConfigFileExtension, "", 1)) } } diff --git a/internal/exec/workflow.go b/internal/exec/workflow.go index 140ac91dd..7c41a573e 100644 --- a/internal/exec/workflow.go +++ b/internal/exec/workflow.go @@ -86,7 +86,7 @@ func ExecuteWorkflowCmd(cmd *cobra.Command, args []string) error { // If the workflow file is specified without an extension, use the default extension ext := filepath.Ext(workflowPath) if ext == "" { - ext = cfg.DefaultStackConfigFileExtension + ext = u.DefaultStackConfigFileExtension workflowPath = workflowPath + ext } diff --git a/pkg/config/const.go b/pkg/config/const.go index 0f3a6af22..8078a1c00 100644 --- a/pkg/config/const.go +++ b/pkg/config/const.go @@ -1,11 +1,9 @@ package config const ( - DefaultStackConfigFileExtension = ".yaml" - DefaultVendoringManifestFileExtension = ".yaml" - CliConfigFileName = "atmos" - SystemDirConfigFilePath = "/usr/local/etc/atmos" - WindowsAppDataEnvVar = "LOCALAPPDATA" + CliConfigFileName = "atmos" + SystemDirConfigFilePath = "/usr/local/etc/atmos" + WindowsAppDataEnvVar = "LOCALAPPDATA" // GlobalOptionsFlag is a custom flag to specify helmfile `GLOBAL OPTIONS` // https://github.com/roboll/helmfile#cli-reference diff --git a/pkg/config/utils.go b/pkg/config/utils.go index ced26646f..93505c634 100644 --- a/pkg/config/utils.go +++ b/pkg/config/utils.go @@ -25,20 +25,28 @@ func FindAllStackConfigsInPathsForStack( var stackIsDir = strings.IndexAny(stack, "/") > 0 for _, p := range includeStackPaths { - pathWithExt := p + // Try both regular and template patterns + patterns := []string{p} ext := filepath.Ext(p) if ext == "" { - ext = DefaultStackConfigFileExtension - pathWithExt = p + ext + patterns = getStackFilePatterns(p) } - // Find all matches in the glob - matches, err := u.GetGlobMatches(pathWithExt) - if err != nil || len(matches) == 0 { - // Retry (b/c we are using `doublestar` library, and it sometimes has issues reading many files in a Docker container) - // TODO: review `doublestar` library - matches, err = u.GetGlobMatches(pathWithExt) + var allMatches []string + for _, pattern := range patterns { + // Find all matches in the glob + matches, err := u.GetGlobMatches(pattern) + if err == nil && len(matches) > 0 { + allMatches = append(allMatches, matches...) + } + } + + // If no matches were found across all patterns, we perform an additional check: + // We try to get matches for the first pattern only to determine if there's a genuine error + // (like permission issues or invalid path) versus simply no matching files. + if len(allMatches) == 0 { + _, err := u.GetGlobMatches(patterns[0]) if err != nil { if cliConfig.Logs.Level == u.LogLevelTrace { y, _ := u.ConvertToYAML(cliConfig) @@ -46,15 +54,17 @@ func FindAllStackConfigsInPathsForStack( } return nil, nil, false, err } - + // If there's no error but still no matches, we continue to the next path + // This happens when the pattern is valid but no files match it + continue } - // Exclude files that match any of the excludePaths - for _, matchedFileAbsolutePath := range matches { + // Process all matches found + for _, matchedFileAbsolutePath := range allMatches { matchedFileRelativePath := u.TrimBasePathFromPath(cliConfig.StacksBaseAbsolutePath+"/", matchedFileAbsolutePath) // Check if the provided stack matches a file in the config folders (excluding the files from `excludeStackPaths`) - stackMatch := strings.HasSuffix(matchedFileAbsolutePath, stack+DefaultStackConfigFileExtension) + stackMatch := matchesStackFilePattern(matchedFileAbsolutePath, stack) if stackMatch { allExcluded := true @@ -93,6 +103,10 @@ func FindAllStackConfigsInPathsForStack( } } + if len(absolutePaths) == 0 { + return nil, nil, false, fmt.Errorf("no matches found for the provided stack '%s' in the paths %v", stack, includeStackPaths) + } + return absolutePaths, relativePaths, false, nil } @@ -107,20 +121,27 @@ func FindAllStackConfigsInPaths( var relativePaths []string for _, p := range includeStackPaths { - pathWithExt := p + patterns := []string{p} ext := filepath.Ext(p) if ext == "" { - ext = DefaultStackConfigFileExtension - pathWithExt = p + ext + patterns = getStackFilePatterns(p) } - // Find all matches in the glob - matches, err := u.GetGlobMatches(pathWithExt) - if err != nil || len(matches) == 0 { - // Retry (b/c we are using `doublestar` library, and it sometimes has issues reading many files in a Docker container) - // TODO: review `doublestar` library - matches, err = u.GetGlobMatches(pathWithExt) + var allMatches []string + for _, pattern := range patterns { + // Find all matches in the glob + matches, err := u.GetGlobMatches(pattern) + if err == nil && len(matches) > 0 { + allMatches = append(allMatches, matches...) + } + } + + // If no matches were found across all patterns, we perform an additional check: + // We try to get matches for the first pattern only to determine if there's a genuine error + // (like permission issues or invalid path) versus simply no matching files. + if len(allMatches) == 0 { + _, err := u.GetGlobMatches(patterns[0]) if err != nil { if cliConfig.Logs.Level == u.LogLevelTrace { y, _ := u.ConvertToYAML(cliConfig) @@ -128,10 +149,13 @@ func FindAllStackConfigsInPaths( } return nil, nil, err } + // If there's no error but still no matches, we continue to the next path + // This happens when the pattern is valid but no files match it + continue } - // Exclude files that match any of the excludePaths - for _, matchedFileAbsolutePath := range matches { + // Process all matches found + for _, matchedFileAbsolutePath := range allMatches { matchedFileRelativePath := u.TrimBasePathFromPath(cliConfig.StacksBaseAbsolutePath+"/", matchedFileAbsolutePath) include := true @@ -630,3 +654,83 @@ func GetStackNameFromContextAndStackNamePattern( return stack, nil } + +// getStackFilePatterns returns a slice of possible file patterns for a given base path +func getStackFilePatterns(basePath string) []string { + return []string{ + basePath + u.DefaultStackConfigFileExtension, + basePath + u.DefaultStackConfigFileExtension + u.TemplateExtension, + basePath + u.YamlTemplateExtension, + basePath + u.YmlTemplateExtension, + } +} + +// matchesStackFilePattern checks if a file path matches any of the valid stack file patterns +func matchesStackFilePattern(filePath, stackName string) bool { + patterns := getStackFilePatterns(stackName) + for _, pattern := range patterns { + if strings.HasSuffix(filePath, pattern) { + return true + } + } + return false +} + +func getConfigFilePatterns(path string, forGlobMatch bool) []string { + if path == "" { + return []string{} + } + ext := filepath.Ext(path) + if ext != "" { + return []string{path} + } + + patterns := []string{ + path + u.DefaultStackConfigFileExtension, + path + u.DefaultStackConfigFileExtension + u.TemplateExtension, + path + u.YamlTemplateExtension, + path + u.YmlTemplateExtension, + } + if !forGlobMatch { + // For direct file search, include the exact path without extension + patterns = append([]string{path}, patterns...) + } + + return patterns +} + +func SearchConfigFile(configPath string, cliConfig schema.CliConfiguration) (string, error) { + // If path already has an extension, verify it exists + if ext := filepath.Ext(configPath); ext != "" { + if _, err := os.Stat(configPath); err == nil { + return configPath, nil + } + return "", fmt.Errorf("specified config file not found: %s", configPath) + } + + dir := filepath.Dir(configPath) + base := filepath.Base(configPath) + + entries, err := os.ReadDir(dir) + if err != nil { + return "", fmt.Errorf("error reading directory %s: %v", dir, err) + } + + // Create a map of existing files for quick lookup + fileMap := make(map[string]bool) + for _, entry := range entries { + if !entry.IsDir() { + fileMap[entry.Name()] = true + } + } + + // Try all patterns in order + patterns := getConfigFilePatterns(base, false) + for _, pattern := range patterns { + if fileMap[pattern] { + return filepath.Join(dir, pattern), nil + } + } + + return "", fmt.Errorf("failed to find a match for the import '%s' ('%s' + '%s')", configPath, dir, base) +} diff --git a/pkg/utils/file_extensions.go b/pkg/utils/file_extensions.go new file mode 100644 index 000000000..9368e68fe --- /dev/null +++ b/pkg/utils/file_extensions.go @@ -0,0 +1,11 @@ +package utils + +const ( + DefaultStackConfigFileExtension = ".yaml" + DefaultVendoringManifestFileExtension = ".yaml" + YamlFileExtension = ".yaml" + YmlFileExtension = ".yml" + YamlTemplateExtension = ".yaml.tmpl" + YmlTemplateExtension = ".yml.tmpl" + TemplateExtension = ".tmpl" +) diff --git a/pkg/utils/file_utils.go b/pkg/utils/file_utils.go index 2cdb7204f..93cb43762 100644 --- a/pkg/utils/file_utils.go +++ b/pkg/utils/file_utils.go @@ -39,8 +39,13 @@ func FileOrDirExists(filename string) bool { // IsYaml checks if the file has YAML extension (does not check file schema, nor validates the file) func IsYaml(file string) bool { - yamlExtensions := []string{".yaml", ".yml"} + yamlExtensions := []string{YamlFileExtension, YmlFileExtension, YamlTemplateExtension, YmlTemplateExtension} ext := filepath.Ext(file) + if ext == ".tmpl" { + // For .tmpl files, we check if the full extension is .yaml.tmpl or .yml.tmpl + baseExt := filepath.Ext(strings.TrimSuffix(file, ext)) + ext = baseExt + ext + } return SliceContainsString(yamlExtensions, ext) } @@ -180,12 +185,12 @@ func IsSocket(path string) (bool, error) { // If the path has a file extension, it checks if the file exists. // If the path does not have a file extension, it checks for the existence of the file with the provided path and the possible config file extensions func SearchConfigFile(path string) (string, bool) { - // check if the provided has a file extension + // check if the provided path has a file extension if filepath.Ext(path) != "" { return path, FileExists(path) } // Define the possible config file extensions - configExtensions := []string{".yaml", ".yml"} + configExtensions := []string{YamlFileExtension, YmlFileExtension, YamlTemplateExtension, YmlTemplateExtension} for _, ext := range configExtensions { filePath := path + ext if FileExists(filePath) { diff --git a/website/docs/core-concepts/stacks/imports.mdx b/website/docs/core-concepts/stacks/imports.mdx index 69e394a7a..90782f7f4 100644 --- a/website/docs/core-concepts/stacks/imports.mdx +++ b/website/docs/core-concepts/stacks/imports.mdx @@ -52,6 +52,23 @@ import: - catalog/file3.YAML # Expicitly load a file with a .YAML extension ``` +### Automatic Template File Detection + +When importing files without specifying an extension, Atmos will now automatically search for and use template versions of the files if they exist. The search order is: + +1. `.yaml` +2. `.yml` +3. `.yaml.tmpl` +4. `.yml.tmpl` + +For example, if you import `catalog/file1`, Atmos will: +1. First look for `catalog/file1.yaml` or `catalog/file1.yml` +2. If found, check if a template version exists (`catalog/file1.yaml.tmpl` or `catalog/file1.yml.tmpl`) +3. Use the template version if it exists, otherwise use the regular YAML file +4. If no files are found, default to using `.yaml` extension + +This feature makes it easier to work with templated configurations as you don't need to explicitly specify the template file extension - Atmos will automatically use the template version when available. + ## Conventions We recommend placing all baseline "imports" in the `stacks/catalog` folder, however, they can exist anywhere.