From abddf146cfc56fd8c5211ee7a6a47c30dfe97dd6 Mon Sep 17 00:00:00 2001 From: "Vinicius C." Date: Tue, 10 Dec 2024 14:56:11 +0000 Subject: [PATCH] Ignore tmpl files to be validated (#830) * fix: ignore tmpl files to be validated (cherry picked from commit 5402068fe92f183b81323f0150bfe442cdee6af2) * docs for validate tmpl exclusion * fix possible paterns * refactor: remove template file exclusion logic from stack config processing --------- Co-authored-by: Erik Osterman (CEO @ Cloud Posse) Co-authored-by: Andriy Knysh --- internal/exec/validate_stacks.go | 11 ++++-- pkg/config/utils.go | 36 +++++++++++-------- .../cli/commands/validate/validate-stacks.mdx | 2 ++ website/docs/core-concepts/stacks/imports.mdx | 9 +++++ .../stacks/templates/templates.mdx | 6 ++++ 5 files changed, 46 insertions(+), 18 deletions(-) diff --git a/internal/exec/validate_stacks.go b/internal/exec/validate_stacks.go index 3a24091c7..ac269dce8 100644 --- a/internal/exec/validate_stacks.go +++ b/internal/exec/validate_stacks.go @@ -113,8 +113,13 @@ func ValidateStacks(cliConfig schema.CliConfiguration) error { // Include (process and validate) all YAML files in the `stacks` folder in all subfolders includedPaths := []string{"**/*"} - // Don't exclude any YAML files for validation - excludedPaths := []string{} + // Don't exclude any YAML files for validation except template files + excludedPaths := []string{ + // Exclude template files from validation since they may contain invalid YAML before being rendered + "**/*.tmpl", + "**/*.yaml.tmpl", + "**/*.yml.tmpl", + } includeStackAbsPaths, err := u.JoinAbsolutePathWithPaths(cliConfig.StacksBaseAbsolutePath, includedPaths) if err != nil { return err @@ -125,7 +130,7 @@ func ValidateStacks(cliConfig schema.CliConfiguration) error { return err } - u.LogDebug(cliConfig, fmt.Sprintf("Validating all YAML files in the '%s' folder and all subfolders\n", + u.LogDebug(cliConfig, fmt.Sprintf("Validating all YAML files in the '%s' folder and all subfolders (excluding template files)\n", path.Join(cliConfig.BasePath, cliConfig.Stacks.BasePath))) for _, filePath := range stackConfigFilesAbsolutePaths { diff --git a/pkg/config/utils.go b/pkg/config/utils.go index 93505c634..929243c3f 100644 --- a/pkg/config/utils.go +++ b/pkg/config/utils.go @@ -30,7 +30,8 @@ func FindAllStackConfigsInPathsForStack( ext := filepath.Ext(p) if ext == "" { - patterns = getStackFilePatterns(p) + // Get all patterns since filtering is done later + patterns = getStackFilePatterns(p, true) } var allMatches []string @@ -125,7 +126,8 @@ func FindAllStackConfigsInPaths( ext := filepath.Ext(p) if ext == "" { - patterns = getStackFilePatterns(p) + // Get all patterns since filtering is done later + patterns = getStackFilePatterns(p, true) } var allMatches []string @@ -656,18 +658,26 @@ func GetStackNameFromContextAndStackNamePattern( } // getStackFilePatterns returns a slice of possible file patterns for a given base path -func getStackFilePatterns(basePath string) []string { - return []string{ - basePath + u.DefaultStackConfigFileExtension, - basePath + u.DefaultStackConfigFileExtension + u.TemplateExtension, - basePath + u.YamlTemplateExtension, - basePath + u.YmlTemplateExtension, +func getStackFilePatterns(basePath string, includeTemplates bool) []string { + patterns := []string{ + basePath + u.YamlFileExtension, + basePath + u.YmlFileExtension, + } + + if includeTemplates { + patterns = append(patterns, + basePath+u.YamlTemplateExtension, + basePath+u.YmlTemplateExtension, + ) } + + return patterns } // matchesStackFilePattern checks if a file path matches any of the valid stack file patterns func matchesStackFilePattern(filePath, stackName string) bool { - patterns := getStackFilePatterns(stackName) + // Always include template files for normal operations (imports, etc.) + patterns := getStackFilePatterns(stackName, true) for _, pattern := range patterns { if strings.HasSuffix(filePath, pattern) { return true @@ -685,12 +695,8 @@ func getConfigFilePatterns(path string, forGlobMatch bool) []string { return []string{path} } - patterns := []string{ - path + u.DefaultStackConfigFileExtension, - path + u.DefaultStackConfigFileExtension + u.TemplateExtension, - path + u.YamlTemplateExtension, - path + u.YmlTemplateExtension, - } + // include template files for normal operations + patterns := getStackFilePatterns(path, true) if !forGlobMatch { // For direct file search, include the exact path without extension patterns = append([]string{path}, patterns...) diff --git a/website/docs/cli/commands/validate/validate-stacks.mdx b/website/docs/cli/commands/validate/validate-stacks.mdx index 8a7cb2376..9a9faaddb 100644 --- a/website/docs/cli/commands/validate/validate-stacks.mdx +++ b/website/docs/cli/commands/validate/validate-stacks.mdx @@ -25,6 +25,8 @@ atmos validate stacks This command validates Atmos stack manifests and checks the following: - All YAML manifest files for YAML errors and inconsistencies + - Note: Template files (`.yaml.tmpl`, `.yml.tmpl`, `.tmpl`) are excluded from validation since they may contain template placeholders that are invalid YAML before being rendered + - Template files are still automatically detected and processed during normal operations (imports, etc.) - All imports: if they are configured correctly, have valid data types, and point to existing manifest files diff --git a/website/docs/core-concepts/stacks/imports.mdx b/website/docs/core-concepts/stacks/imports.mdx index 90782f7f4..210fa5b6e 100644 --- a/website/docs/core-concepts/stacks/imports.mdx +++ b/website/docs/core-concepts/stacks/imports.mdx @@ -69,6 +69,15 @@ For example, if you import `catalog/file1`, Atmos will: This feature makes it easier to work with templated configurations as you don't need to explicitly specify the template file extension - Atmos will automatically use the template version when available. +:::note Template File Validation +While template files are automatically detected and processed during normal operations (imports, etc.), they are excluded from YAML validation (`atmos validate stacks`) since they may contain template placeholders that are invalid YAML before being rendered. + +This means: +- Template files are fully supported for imports and normal operations +- Template files are skipped during `atmos validate stacks` to prevent validation errors from unrendered templates +- You don't need to explicitly specify template extensions - Atmos will find them automatically +::: + ## Conventions We recommend placing all baseline "imports" in the `stacks/catalog` folder, however, they can exist anywhere. diff --git a/website/docs/core-concepts/stacks/templates/templates.mdx b/website/docs/core-concepts/stacks/templates/templates.mdx index ad6781b8b..4d70dd31f 100644 --- a/website/docs/core-concepts/stacks/templates/templates.mdx +++ b/website/docs/core-concepts/stacks/templates/templates.mdx @@ -20,6 +20,12 @@ import SecondaryCTA from '@site/src/components/SecondaryCTA' Atmos supports [Go templates](https://pkg.go.dev/text/template) in stack manifests and functions to customize Stack configurations. +:::note Template File Validation +Template files (`.yaml.tmpl`, `.yml.tmpl`, `.tmpl`) are automatically detected and processed during normal operations (imports, etc.). +However, they are excluded from YAML validation (`atmos validate stacks`) since they may contain template placeholders that are invalid YAML before being rendered. +This ensures that template files can contain valid Go template syntax without causing validation errors. +::: + ### Enable Templating Templating in Atmos stack manifests is configured in the `atmos.yaml` [CLI config file](/cli/configuration) in the