Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refine Atmos Configuration Handling to Support Imports, Remote Imports, and Configuration Directories #808

Open
wants to merge 72 commits into
base: main
Choose a base branch
from
Open
Changes from 1 commit
Commits
Show all changes
72 commits
Select commit Hold shift + click to select a range
892cfeb
feat: add support for importing configurations from local and remote …
haitham911 Nov 24, 2024
052673b
refactor: streamline configuration initialization and enhance import …
haitham911 Nov 24, 2024
63ffce1
fix: sort full matches instead of initial matches in GetGlobMatches f…
haitham911 Nov 24, 2024
000e439
fix: add json and mapstructure tags to Import field in CliConfiguration
haitham911 Nov 24, 2024
4493540
fix: handle unexpected errors when checking for atmos.d path in InitC…
haitham911 Nov 24, 2024
d1903c4
fix: update atmos.yaml to include wildcard import for YAML files and …
haitham911 Nov 24, 2024
d9a3517
fix: remove obsolete configuration files and update atmos.yaml for cl…
haitham911 Nov 24, 2024
089a82f
fix: validate import paths to prevent directory traversal in InitCliC…
haitham911 Nov 24, 2024
c020d63
fix: enhance downloadRemoteConfig to return temporary directory and h…
haitham911 Nov 24, 2024
31c844c
fix: remove environment variable from extra-config.yaml for cleaner c…
haitham911 Nov 30, 2024
d70f66f
feat: update YAML configurations to define commands and enhance compo…
haitham911 Dec 3, 2024
512e02e
fix: replace path package with filepath for consistent path handling
haitham911 Dec 3, 2024
c22624f
Merge branch 'main' into DEV-1534
osterman Dec 4, 2024
750f4b5
feat: enhance CLI configuration loading logic to prioritize ATMOS_CLI…
haitham911 Dec 7, 2024
8eeda00
feat: reorganize configuration files and enhance YAML support for log…
haitham911 Dec 11, 2024
a8d3528
feat: rename environment variable for clarity and improve test handli…
haitham911 Dec 11, 2024
d8a00ef
feat: update remote configuration URL in atmos.yaml for improved acce…
haitham911 Dec 11, 2024
8050589
feat: restructure configuration files and remove deprecated logging s…
haitham911 Dec 11, 2024
52b170d
feat: migrate custom import configuration files to a new structure fo…
haitham911 Dec 11, 2024
f85d53e
feat: remove deprecated custom import configuration files and update …
haitham911 Dec 11, 2024
73d9255
feat: add URL validation for remote imports to enhance error handling
haitham911 Dec 12, 2024
9913b54
feat: update configuration documentation to clarify CLI config loadin…
haitham911 Dec 12, 2024
cdb9fa1
fix: correct example paths in CLI configuration documentation for cla…
haitham911 Dec 12, 2024
7090975
fix: clarify environment variable usage in CLI configuration document…
haitham911 Dec 12, 2024
3dfb2e0
Update examples/demo-atmos-cli-imports/configs.d/vendor.yaml
osterman Dec 12, 2024
cdafe20
enhance CLI configuration documentation with detailed import examples…
haitham911 Dec 13, 2024
b702884
fix: improve atmos.d path validation and add support for .atmos.d dir…
haitham911 Dec 13, 2024
8704668
fix: update command in demo configuration and improve URL validation …
haitham911 Dec 13, 2024
989cd9e
Merge branch 'main' into DEV-1534
haitham911 Dec 17, 2024
a7414bb
add env demo
haitham911 Dec 17, 2024
3a24ee6
expose env variable ATMOS_CLI_CONFIG_PATH and ATMOS_BASE_PATH before…
haitham911 Dec 18, 2024
061ba9c
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 20, 2024
e57aacb
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 20, 2024
e6a3200
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 20, 2024
fa5f25e
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 20, 2024
86e2505
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 20, 2024
95cdf74
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 20, 2024
bc2c4bc
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 20, 2024
8e6998a
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 20, 2024
87d4b0f
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 20, 2024
31d71c1
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 20, 2024
e64d2eb
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 20, 2024
378897c
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 20, 2024
e8d8507
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 20, 2024
6e4bc4d
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 20, 2024
930fde6
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 20, 2024
94e99d6
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 20, 2024
85957a4
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 20, 2024
b75c251
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 20, 2024
a742412
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 20, 2024
30c82d2
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 20, 2024
0609a78
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 20, 2024
1c02902
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 20, 2024
792ff0a
Apply suggestions from code review
osterman Dec 20, 2024
33da5d0
Apply suggestions from code review
osterman Dec 20, 2024
885bd00
Apply suggestions from code review
osterman Dec 20, 2024
b933331
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 20, 2024
50cd50e
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 30, 2024
aafdf16
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 30, 2024
3bb33fa
Update examples/demo-env/stacks/deploy/prod.yaml
osterman Dec 30, 2024
a55f6f2
Update website/docs/cli/configuration/configuration.mdx
osterman Dec 30, 2024
66c0a59
Apply suggestions from code review
osterman Dec 30, 2024
27d3acd
Apply suggestions from code review
osterman Dec 30, 2024
1d46232
revise the docs for intended behavior
osterman Dec 30, 2024
9a27195
Make it more clear
osterman Dec 30, 2024
bce6c4d
Merge branch 'main' into DEV-1534
haitham911 Jan 5, 2025
f45cf0d
fix missed from marge main branch
haitham911 Jan 5, 2025
1d33e16
change to atmosConfig
haitham911 Jan 5, 2025
f9e44cd
replace cliConfig with atmosConfig
haitham911 Jan 5, 2025
086c8c6
add mermaid for configuration
haitham911 Jan 5, 2025
bf02e59
edit mermaid doc
haitham911 Jan 5, 2025
eb443e3
refactor updates
haitham911 Jan 16, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
feat: enhance CLI configuration loading logic to prioritize ATMOS_CLI…
…_CONFIG_PATH and streamline fallback paths
  • Loading branch information
haitham911 committed Dec 7, 2024
commit 750f4b52062020e4f863d5c847b5b471117deeaa
155 changes: 81 additions & 74 deletions pkg/config/config.go
Original file line number Diff line number Diff line change
@@ -93,16 +93,18 @@ var (
// https://medium.com/@bnprashanth256/reading-configuration-files-and-environment-variables-in-go-golang-c2607f912b63
func InitCliConfig(configAndStacksInfo schema.ConfigAndStacksInfo, processStacks bool) (schema.CliConfiguration, error) {
// cliConfig is loaded from the following locations (from lower to higher priority):
// system dir (`/usr/local/etc/atmos` on Linux, `%LOCALAPPDATA%/atmos` on Windows)
// home dir (~/.atmos)
// current directory
// ENV vars
// Command-line arguments

// 1. If ATMOS_CLI_CONFIG_PATH is defined, check only there
// 2. If ATMOS_CLI_CONFIG_PATH is not defined, proceed with other paths
// - Check system directory (optional)
// - Check user-specific configuration:
// - If XDG_CONFIG_HOME is defined, use it; otherwise fallback to ~/.config/atmos
// - Check current directory
// - If Terraform provider specified a path
// 3. If no config is found in any of the above locations, use the default config
// Check if no imports are defined
var cliConfig schema.CliConfiguration
var err error
configFound := false
var found bool

v := viper.New()
v.SetConfigType("yaml")
@@ -112,81 +114,88 @@ func InitCliConfig(configAndStacksInfo schema.ConfigAndStacksInfo, processStacks
v.SetDefault("components.helmfile.use_eks", true)
v.SetDefault("components.terraform.append_user_agent", fmt.Sprintf("Atmos/%s (Cloud Posse; +https://atmos.tools)", version.Version))

// Process config in system folder
configFilePath1 := ""

// https://pureinfotech.com/list-environment-variables-windows-10/
// https://docs.microsoft.com/en-us/windows/deployment/usmt/usmt-recognized-environment-variables
// https://softwareengineering.stackexchange.com/questions/299869/where-is-the-appropriate-place-to-put-application-configuration-files-for-each-p
// https://stackoverflow.com/questions/37946282/why-does-appdata-in-windows-7-seemingly-points-to-wrong-folder
if runtime.GOOS == "windows" {
appDataDir := os.Getenv(WindowsAppDataEnvVar)
if len(appDataDir) > 0 {
configFilePath1 = appDataDir
// 1. If ATMOS_CLI_CONFIG_PATH is defined, check only there
if atmEnvPath := os.Getenv("ATMOS_CLI_CONFIG_PATH"); atmEnvPath != "" {
haitham911 marked this conversation as resolved.
Show resolved Hide resolved
u.LogTrace(cliConfig, fmt.Sprintf("Found ENV var ATMOS_CLI_CONFIG_PATH=%s", atmEnvPath))
configFile := filepath.Join(atmEnvPath, CliConfigFileName)
found, err := processConfigFile(cliConfig, configFile, v)
if err != nil {
return cliConfig, err
}
if !found {
// If we want to error out if config not found in ATMOS_CLI_CONFIG_PATH
return cliConfig, fmt.Errorf("config not found in ATMOS_CLI_CONFIG_PATH: %s", configFile)
} else {
configFound = true
}

// Since ATMOS_CLI_CONFIG_PATH is to be the first and only check, we skip other paths if found
// If not found, we still skip other paths as per the requirement.
} else {
configFilePath1 = SystemDirConfigFilePath
}
// 2. If ATMOS_CLI_CONFIG_PATH is not defined, proceed with other paths
// - Check system directory (optional)
configFilePathSystem := ""
if runtime.GOOS == "windows" {
appDataDir := os.Getenv(WindowsAppDataEnvVar)
if len(appDataDir) > 0 {
configFilePathSystem = appDataDir
}
} else {
configFilePathSystem = SystemDirConfigFilePath
}

if len(configFilePathSystem) > 0 {
configFile := filepath.Join(configFilePathSystem, CliConfigFileName)
found, err := processConfigFile(cliConfig, configFile, v)
if err != nil {
return cliConfig, err
}
if found {
configFound = true
}
}

// 3. Check user-specific configuration:
// If XDG_CONFIG_HOME is defined, use it; otherwise fallback to ~/.config/atmos
xdgConfigHome := os.Getenv("XDG_CONFIG_HOME")
var userConfigDir string
if xdgConfigHome != "" {
userConfigDir = filepath.Join(xdgConfigHome, "atmos")
} else {
homeDir, err := homedir.Dir()
if err != nil {
return cliConfig, err
}
userConfigDir = filepath.Join(homeDir, ".config", "atmos")
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Use xdg go library


if len(configFilePath1) > 0 {
configFile1 := filepath.Join(configFilePath1, CliConfigFileName)
found, err = processConfigFile(cliConfig, configFile1, v)
userConfigFile := filepath.Join(userConfigDir, CliConfigFileName)
found, err := processConfigFile(cliConfig, userConfigFile, v)
if err != nil {
return cliConfig, err
}
if found {
configFound = true
}
}

// Process config in user's HOME dir
configFilePath2, err := homedir.Dir()
if err != nil {
return cliConfig, err
}
configFile2 := filepath.Join(configFilePath2, ".atmos", CliConfigFileName)
found, err = processConfigFile(cliConfig, configFile2, v)
if err != nil {
return cliConfig, err
}
if found {
configFound = true
}

// Process config in the current dir
configFilePath3, err := os.Getwd()
if err != nil {
return cliConfig, err
}
configFile3 := filepath.Join(configFilePath3, CliConfigFileName)
found, err = processConfigFile(cliConfig, configFile3, v)
if err != nil {
return cliConfig, err
}
if found {
configFound = true
}

// Process config from the path in ENV var `ATMOS_CLI_CONFIG_PATH`
configFilePath4 := os.Getenv("ATMOS_CLI_CONFIG_PATH")
if len(configFilePath4) > 0 {
u.LogTrace(cliConfig, fmt.Sprintf("Found ENV var ATMOS_CLI_CONFIG_PATH=%s", configFilePath4))
configFile4 := filepath.Join(configFilePath4, CliConfigFileName)
found, err = processConfigFile(cliConfig, configFile4, v)
// 4. Check current directory
configFilePathCwd, err := os.Getwd()
if err != nil {
return cliConfig, err
}
configFileCwd := filepath.Join(configFilePathCwd, CliConfigFileName)
found, err = processConfigFile(cliConfig, configFileCwd, v)
if err != nil {
return cliConfig, err
}
if found {
configFound = true
}
}

// Process config from the path specified in the Terraform provider (which calls into the atmos code)
if configAndStacksInfo.AtmosCliConfigPath != "" {
configFilePath5 := configAndStacksInfo.AtmosCliConfigPath
if len(configFilePath5) > 0 {
configFile5 := filepath.Join(configFilePath5, CliConfigFileName)
found, err = processConfigFile(cliConfig, configFile5, v)
// 5. If Terraform provider specified a path
if configAndStacksInfo.AtmosCliConfigPath != "" {
configFileTfProvider := filepath.Join(configAndStacksInfo.AtmosCliConfigPath, CliConfigFileName)
found, err := processConfigFile(cliConfig, configFileTfProvider, v)
if err != nil {
return cliConfig, err
}
@@ -197,12 +206,11 @@ func InitCliConfig(configAndStacksInfo schema.ConfigAndStacksInfo, processStacks
}

if !configFound {
// If `atmos.yaml` not found, use the default config
// Set `ATMOS_LOGS_LEVEL` ENV var to "Debug" to see the message about Atmos using the default CLI config
// Use default config if no config was found in any location
logsLevelEnvVar := os.Getenv("ATMOS_LOGS_LEVEL")
if logsLevelEnvVar == u.LogLevelDebug || logsLevelEnvVar == u.LogLevelTrace {
u.PrintMessageInColor("'atmos.yaml' CLI config was not found in any of the searched paths: system dir, home dir, current dir, ENV vars.\n"+
"Refer to https://atmos.tools/cli/configuration for details on how to configure 'atmos.yaml'.\n"+
u.PrintMessageInColor("'atmos.yaml' CLI config was not found.\n"+
"Refer to https://atmos.tools/cli/configuration\n"+
"Using the default CLI config:\n\n", color.New(color.FgCyan))
haitham911 marked this conversation as resolved.
Show resolved Hide resolved

err = u.PrintAsYAMLToFileDescriptor(cliConfig, defaultCliConfig)
@@ -224,18 +232,17 @@ func InitCliConfig(configAndStacksInfo schema.ConfigAndStacksInfo, processStacks
}
}

// https://gist.github.com/chazcheadle/45bf85b793dea2b71bd05ebaa3c28644
// https://sagikazarmark.hu/blog/decoding-custom-formats-with-viper/
// Unmarshal, process environment variables, imports, and command-line arguments as needed.
err = v.Unmarshal(&cliConfig)
if err != nil {
return cliConfig,
err
return cliConfig, err
}
// Process ENV vars

err = processEnvVars(&cliConfig)
if err != nil {
return cliConfig, err
}

// Check if no imports are defined
if len(cliConfig.Import) == 0 {
basePath, err := filepath.Abs(cliConfig.BasePath)
134 changes: 134 additions & 0 deletions pkg/config/config_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
package config

import (
"os"
"path/filepath"
"testing"

"github.com/stretchr/testify/require"

"github.com/cloudposse/atmos/pkg/schema"
)

// Config loads successfully from ATMOS_CLI_CONFIG_PATH when env var is set
func TestInitCliConfigLoadsFromAtmosCliConfigPath(t *testing.T) {
// Setup test directory and config file
tmpDir := t.TempDir()
configPath := filepath.Join(tmpDir, "atmos.yaml")

// Create test config content
configContent := []byte(`
components:
terraform:
base_path: terraform
stacks:
base_path: "stacks"
included_paths:
- "deploy/**/*"
excluded_paths:
- "**/_defaults.yaml"
name_pattern: "{stage}"
`)
err := os.WriteFile(configPath, configContent, 0644)
require.NoError(t, err)

// Set env var to temp config path
t.Setenv("ATMOS_CLI_CONFIG_PATH", tmpDir)

// Create test input
configInfo := schema.ConfigAndStacksInfo{
Stack: "test-stack",
}

// Call function under test
cfg, err := InitCliConfig(configInfo, false)
require.NoError(t, err)

// Assert results
require.True(t, cfg.Initialized)
require.Equal(t, "terraform", cfg.Components.Terraform.BasePath)
}

// Empty or invalid ATMOS_CLI_CONFIG_PATH environment variable
func TestInitCliConfigWithInvalidEnvPath(t *testing.T) {
// Set env var to non-existent path
t.Setenv("ATMOS_CLI_CONFIG_PATH", "/non/existent/path")

// Create test input
configInfo := schema.ConfigAndStacksInfo{
Stack: "test-stack",
}

// Call function under test
_, err := InitCliConfig(configInfo, false)

// Assert error is returned
require.Error(t, err)
require.Contains(t, err.Error(), "config not found in ATMOS_CLI_CONFIG_PATH")
}

// Config loads from default locations when ATMOS_CLI_CONFIG_PATH is not set
func TestConfigLoadsFromDefaultLocations(t *testing.T) {
os.Unsetenv("ATMOS_CLI_CONFIG_PATH")

// Setup temporary directory and default config file
tmpDir := t.TempDir()
configPath := filepath.Join(tmpDir, "atmos.yaml")
configContent := []byte(`
components:
helmfile:
use_eks: true
terraform:
append_user_agent: Atmos/1.0.0 (Cloud Posse; +https://atmos.tools)
stacks:
base_path: "stacks"
included_paths:
- "deploy/**/*"
excluded_paths:
- "**/_defaults.yaml"
name_pattern: "{stage}"
`)
err := os.WriteFile(configPath, configContent, 0644)
require.NoError(t, err)

// Change working directory to temporary directory
err = os.Chdir(tmpDir)
require.NoError(t, err)

configAndStacksInfo := schema.ConfigAndStacksInfo{}
cliConfig, err := InitCliConfig(configAndStacksInfo, false)
require.NoError(t, err)
require.True(t, cliConfig.Initialized)
}
haitham911 marked this conversation as resolved.
Show resolved Hide resolved

// Imports from atmos.d directory are processed automatically when no explicit imports defined
func TestImportsFromAtmosDProcessedAutomatically(t *testing.T) {
os.Unsetenv("ATMOS_CLI_CONFIG_PATH")

// Setup temporary directory and atmos.d
tmpDir := t.TempDir()
atmosDPath := filepath.Join(tmpDir, "atmos.d")
err := os.Mkdir(atmosDPath, 0755)
require.NoError(t, err)

// Create a sample import file
importFilePath := filepath.Join(atmosDPath, "sample.yaml")
importContent := []byte(`
imports:
- some/import/path.yaml
`)
err = os.WriteFile(importFilePath, importContent, 0644)
require.NoError(t, err)

// Change working directory to temporary directory
err = os.Chdir(tmpDir)
require.NoError(t, err)

configAndStacksInfo := schema.ConfigAndStacksInfo{}
cliConfig, err := InitCliConfig(configAndStacksInfo, false)
require.NoError(t, err)

if len(cliConfig.Import) == 0 {
t.Fatalf("Expected imports to be processed from atmos.d directory")
}
}
Loading