Supported the atmos.d, allowing automatic inclusion of configuration files from the atmos.d directory

examples/demo-atmos.d/atmos.d/config1.yaml

@@ -0,0 +1,4 @@
+logs:
+  level: Debug
+settings:
+  retries: 3

examples/demo-atmos.d/atmos.d/sub/config2.yaml

@@ -0,0 +1,5 @@
+logs:
+  level: Warning
+settings:
+  timeout: 30s
+

examples/demo-atmos.d/atmos.yaml

@@ -0,0 +1,41 @@
+# Description: Configuration file for the Atmos CLI
+# default import path is ./atmos.d import .yaml files from the directory and merge them
+base_path: "./"
+components:
+  terraform:
+    base_path: "components/terraform"
+    apply_auto_approve: false
+    deploy_run_init: true
+    init_run_reconfigure: true
+    auto_generate_backend_file: false
+
+stacks:
+  base_path: "stacks"
+  included_paths:
+    - "deploy/**/*"
+  excluded_paths:
+    - "**/_defaults.yaml"
+  name_pattern: "{stage}"
+
+vendor:  
+  # Single file
+  base_path: "./vendor.yaml"
+
+  # Directory with multiple files
+  #base_path: "./vendor"
+
+  # Absolute path
+  #base_path: "vendor.d/vendor1.yaml"
+
+logs:
+  file: "/dev/stderr"
+  level: Info
+
+# Custom CLI commands
+
+# No arguments or flags are required
+commands:
+- name: "test"
+  description: "Run all tests"
+  steps:
+  - atmos vendor pull

examples/demo-atmos.d/custom-import/atmos.yaml

@@ -0,0 +1,47 @@
+# Description: This is an example of a custom import configuration file.
+# The import configuration file is used to load configurations from multiple files and directories.
+# The configurations are merged together to create a single configuration object.
+# The configurations are loaded in the order they are defined in the import section.
+base_path: "./"
+import:
+  - "configs.d/**/*"          # Recursively load configurations from a directory
+  - "https://example.com/config.yaml" # Load from a remote URL
+  - "./extra-config.yaml"     # Load a specific file
+components:
+  terraform:
+    base_path: "components/terraform"
+    apply_auto_approve: false
+    deploy_run_init: true
+    init_run_reconfigure: true
+    auto_generate_backend_file: false
+
+stacks:
+  base_path: "stacks"
+  included_paths:
+    - "deploy/**/*"
+  excluded_paths:
+    - "**/_defaults.yaml"
+  name_pattern: "{stage}"
+
+vendor:  
+  # Single file
+  base_path: "./vendor.yaml"
+
+  # Directory with multiple files
+  #base_path: "./vendor"
+
+  # Absolute path
+  #base_path: "vendor.d/vendor1.yaml"
+
+logs:
+  file: "/dev/stderr"
+  level: Info
+
+# Custom CLI commands
+
+# No arguments or flags are required
+commands:
+  - name: "test"
+    description: "Run all tests"
+    steps:
+      - atmos vendor pull

examples/demo-atmos.d/custom-import/configs.d/config1.yaml

@@ -0,0 +1,4 @@
+logs:
+  level: Debug
+settings:
+  retries: 3

examples/demo-atmos.d/custom-import/configs.d/sub/config2.yaml

@@ -0,0 +1,4 @@
+logs:
+  level: Warning
+settings:
+  timeout: 30s

examples/demo-atmos.d/custom-import/extra-config.yaml

@@ -0,0 +1,3 @@
+environment: production
+logs:
+  file: /var/log/atmos.log

pkg/config/config.go

@@ -2,14 +2,17 @@ package config
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"os"
 	"path"
 	"path/filepath"
 	"runtime"
+	"strings"
 
 	"github.com/fatih/color"
+	"github.com/hashicorp/go-getter"
 	"github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	"github.com/spf13/viper"
@@ -225,14 +228,45 @@ func InitCliConfig(configAndStacksInfo schema.ConfigAndStacksInfo, processStacks
 	// https://sagikazarmark.hu/blog/decoding-custom-formats-with-viper/
 	err = v.Unmarshal(&cliConfig)
 	if err != nil {
-		return cliConfig, err
+		return cliConfig,
+			err
 	}
-
 	// Process ENV vars
 	err = processEnvVars(&cliConfig)
 	if err != nil {
 		return cliConfig, err
 	}
+	// Check if 'import' key exists
+	if len(cliConfig.Import) == 0 {
+		basePath, err := filepath.Abs(cliConfig.BasePath)
+		if err != nil {
+			return cliConfig, err
+		}
+		atmosDPath := path.Join(basePath, "atmos.d")
+		// Ensure the joined path doesn't escape the intended directory
+		if !strings.HasPrefix(atmosDPath, basePath) {
+			return cliConfig, fmt.Errorf("invalid atmos.d path: attempted directory traversal")
+		}
+		_, err = os.Stat(atmosDPath)
+		if err == nil {
+			cliConfig.Import = []string{"atmos.d/**/*.yaml"}
+		} else if !os.IsNotExist(err) {
+			return cliConfig, err // Handle unexpected errors
+		}
+	}
+	// Process imports if any
+	if len(cliConfig.Import) > 0 {
+		err = processImports(cliConfig, v)
+		if err != nil {
+			return cliConfig, err
+		}
+
+		// Re-unmarshal the merged configuration into cliConfig
+		err = v.Unmarshal(&cliConfig)
+		if err != nil {
+			return cliConfig, err
+		}
+	}
 
 	// Process command-line args
 	err = processCommandLineArgs(&cliConfig, configAndStacksInfo)
@@ -370,3 +404,76 @@ func processConfigFile(
 
 	return true, nil
 }
+func processImports(cliConfig schema.CliConfiguration, v *viper.Viper) error {
+	for _, importPath := range cliConfig.Import {
+		if importPath == "" {
+			continue
+		}
+
+		var resolvedPaths []string
+		var err error
+
+		if strings.HasPrefix(importPath, "http://") || strings.HasPrefix(importPath, "https://") {
+			// Handle remote URLs
+			tempFile, err := downloadRemoteConfig(importPath)
+			if err != nil {
+				u.LogWarning(cliConfig, fmt.Sprintf("Warning: failed to download remote config '%s': %v", importPath, err))
+				continue
+			}
+			resolvedPaths = []string{tempFile}
+		} else {
+			impWithExt := importPath
+			ext := filepath.Ext(importPath)
+			if ext == "" {
+				ext = ".yaml"
+				impWithExt = importPath + ext
+			}
+			basePath, err := filepath.Abs(cliConfig.BasePath)
+			if err != nil {
+				return err
+			}
+			imp := path.Join(basePath, impWithExt)
+			// ensure the joined path doesn't escape the intended directory
+			if !strings.HasPrefix(imp, basePath) {
+				return fmt.Errorf("invalid import path: attempted directory traversal")
+			}
+			resolvedPaths, err = u.GetGlobMatches(imp)
+			if err != nil {
+				u.LogWarning(cliConfig, fmt.Sprintf("Warning: failed to resolve import path '%s': %v", impWithExt, err))
+				continue
+			}
+		}
+		// print the resolved paths
+		u.LogTrace(cliConfig, fmt.Sprintf("Resolved import paths: %v", resolvedPaths))
+		for _, path := range resolvedPaths {
+			// Process each configuration file
+			_, err = processConfigFile(cliConfig, path, v)
+			if err != nil {
+				// Log the error but continue processing other files
+				u.LogWarning(cliConfig, fmt.Sprintf("Warning: failed to merge configuration from '%s': %v", path, err))
+				continue
+			}
+		}
+	}
+	return nil
+}
+
+func downloadRemoteConfig(url string) (string, error) {
+	tempDir, err := os.MkdirTemp("", "atmos-import-*")
+	if err != nil {
+		return "", err
+	}
+	tempFile := filepath.Join(tempDir, "config.yaml")
+
+	client := &getter.Client{
+		Ctx:  context.Background(),
+		Src:  url,
+		Dst:  tempFile,
+		Mode: getter.ClientModeFile,
+	}
+	err = client.Get()
+	if err != nil {
+		return "", fmt.Errorf("failed to download remote config: %w", err)
+	}
+	return tempFile, nil
+}

pkg/schema/schema.go

@@ -26,6 +26,7 @@ type CliConfiguration struct {
 	StackConfigFilesAbsolutePaths []string       `yaml:"stackConfigFilesAbsolutePaths,omitempty" json:"stackConfigFilesAbsolutePaths,omitempty" mapstructure:"stackConfigFilesAbsolutePaths"`
 	StackType                     string         `yaml:"stackType,omitempty" json:"StackType,omitempty" mapstructure:"stackType"`
 	Default                       bool           `yaml:"default" json:"default" mapstructure:"default"`
+	Import                        []string       `yaml:"import" json:"import" mapstructure:"import"`
 }
 
 type CliSettings struct {

pkg/utils/glob_utils.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"sort"
 	"strings"
 	"sync"
 
@@ -38,6 +39,8 @@ func GetGlobMatches(pattern string) ([]string, error) {
 	for _, match := range matches {
 		fullMatches = append(fullMatches, path.Join(base, match))
 	}
+	// Sort matches lexicographically
+	sort.Strings(fullMatches)
 
 	getGlobMatchesSyncMap.Store(pattern, strings.Join(fullMatches, ","))