Skip to content

Commit

Permalink
atmos read storage interface (#865)
Browse files Browse the repository at this point in the history
  • Loading branch information
mcalhoun authored and Listener430 committed Dec 18, 2024
1 parent c78ab33 commit b75be94
Show file tree
Hide file tree
Showing 19 changed files with 904 additions and 64 deletions.
30 changes: 17 additions & 13 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,16 @@ require (
github.com/Masterminds/sprig/v3 v3.3.0
github.com/alecthomas/chroma v0.10.0
github.com/arsham/figurine v1.3.0
github.com/aws/aws-sdk-go-v2 v1.32.6
github.com/aws/aws-sdk-go-v2/config v1.28.6
github.com/aws/aws-sdk-go-v2/service/ssm v1.56.1
github.com/bmatcuk/doublestar/v4 v4.7.1
github.com/charmbracelet/bubbles v0.20.0
github.com/charmbracelet/bubbletea v1.2.4
github.com/charmbracelet/glamour v0.8.0
github.com/charmbracelet/huh v0.6.0
github.com/charmbracelet/lipgloss v1.0.0
github.com/charmbracelet/log v0.4.0
github.com/elewis787/boa v0.1.2
github.com/fatih/color v1.18.0
github.com/go-git/go-git/v5 v5.12.0
Expand Down Expand Up @@ -72,24 +76,23 @@ require (
github.com/arsham/rainbow v1.2.1 // indirect
github.com/atotto/clipboard v0.1.4 // indirect
github.com/aws/aws-sdk-go v1.44.206 // indirect
github.com/aws/aws-sdk-go-v2 v1.16.4 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1 // indirect
github.com/aws/aws-sdk-go-v2/config v1.15.9 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.12.4 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.5 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.47 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.14 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.11 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.12 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.6 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.5 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.26.10 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.7 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.16.6 // indirect
github.com/aws/smithy-go v1.11.2 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 // indirect
github.com/aws/smithy-go v1.22.1 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
Expand Down Expand Up @@ -126,6 +129,7 @@ require (
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.5.0 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/go-logfmt/logfmt v0.6.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/gobwas/glob v0.2.3 // indirect
Expand Down Expand Up @@ -244,7 +248,7 @@ require (
go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2 // indirect
gocloud.dev v0.25.1-0.20220408200107-09b10f7359f7 // indirect
golang.org/x/crypto v0.28.0 // indirect
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
golang.org/x/mod v0.18.0 // indirect
golang.org/x/net v0.30.0 // indirect
golang.org/x/oauth2 v0.22.0 // indirect
Expand Down
301 changes: 287 additions & 14 deletions go.sum

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion internal/exec/yaml_func_exec.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ func processTagExec(
) any {
u.LogTrace(cliConfig, fmt.Sprintf("Executing Atmos YAML function: %s", input))

str, err := getStringAfterTag(cliConfig, input, config.AtmosYamlFuncExec)
str, err := getStringAfterTag(input, config.AtmosYamlFuncExec)

if err != nil {
u.LogErrorAndExit(cliConfig, err)
Expand Down
41 changes: 41 additions & 0 deletions internal/exec/yaml_func_store.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package exec

import (
"fmt"
"strings"

"github.com/charmbracelet/log"
"github.com/cloudposse/atmos/pkg/schema"
u "github.com/cloudposse/atmos/pkg/utils"
)

func processTagStore(cliConfig schema.CliConfiguration, input string, currentStack string) any {
log.Debug("Executing Atmos YAML function store", "input", input)

str, err := getStringAfterTag(input, u.AtmosYamlFuncStore)

if err != nil {
u.LogErrorAndExit(cliConfig, err)
}

parts := strings.Split(str, " ")
if len(parts) != 2 {
u.LogErrorAndExit(cliConfig, fmt.Errorf("invalid Atmos Store YAML function execution:: %s\nexactly two parameters are required: store_name, key", input))
}

storeName := strings.TrimSpace(parts[0])
key := strings.TrimSpace(parts[1])

store := cliConfig.Stores[storeName]

if store == nil {
u.LogErrorAndExit(cliConfig, fmt.Errorf("invalid Atmos Store YAML function execution:: %s\nstore '%s' not found", input, storeName))
}

value, err := store.Get(key)
if err != nil {
u.LogErrorAndExit(cliConfig, fmt.Errorf("invalid Atmos Store YAML function execution:: %s\nkey '%s' not found in store '%s'", input, key, storeName))
}

return value
}
2 changes: 1 addition & 1 deletion internal/exec/yaml_func_template.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ func processTagTemplate(
) any {
u.LogTrace(cliConfig, fmt.Sprintf("Executing Atmos YAML function: %s", input))

str, err := getStringAfterTag(cliConfig, input, config.AtmosYamlFuncTemplate)
str, err := getStringAfterTag(input, config.AtmosYamlFuncTemplate)

if err != nil {
u.LogErrorAndExit(cliConfig, err)
Expand Down
2 changes: 1 addition & 1 deletion internal/exec/yaml_func_terraform_output.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ func processTagTerraformOutput(
) any {
u.LogTrace(cliConfig, fmt.Sprintf("Executing Atmos YAML function: %s", input))

str, err := getStringAfterTag(cliConfig, input, config.AtmosYamlFuncTerraformOutput)
str, err := getStringAfterTag(input, config.AtmosYamlFuncTerraformOutput)
if err != nil {
u.LogErrorAndExit(cliConfig, err)
}
Expand Down
20 changes: 12 additions & 8 deletions internal/exec/yaml_func_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ import (
"fmt"
"strings"

"github.com/cloudposse/atmos/pkg/config"
"github.com/cloudposse/atmos/pkg/schema"
u "github.com/cloudposse/atmos/pkg/utils"
)

func ProcessCustomYamlTags(
Expand Down Expand Up @@ -60,19 +60,23 @@ func processCustomTags(
input string,
currentStack string,
) any {
if strings.HasPrefix(input, config.AtmosYamlFuncTemplate) {

switch {
case strings.HasPrefix(input, u.AtmosYamlFuncTemplate):
return processTagTemplate(cliConfig, input, currentStack)
} else if strings.HasPrefix(input, config.AtmosYamlFuncExec) {
case strings.HasPrefix(input, u.AtmosYamlFuncExec):
return processTagExec(cliConfig, input, currentStack)
} else if strings.HasPrefix(input, config.AtmosYamlFuncTerraformOutput) {
case strings.HasPrefix(input, u.AtmosYamlFuncStore):
return processTagStore(cliConfig, input, currentStack)
case strings.HasPrefix(input, u.AtmosYamlFuncTerraformOutput):
return processTagTerraformOutput(cliConfig, input, currentStack)
default:
// If any other YAML explicit type (not currently supported by Atmos) is used, return it w/o processing
return input
}

// If any other YAML explicit type (not currently supported by Atmos) is used, return it w/o processing
return input
}

func getStringAfterTag(cliConfig schema.CliConfiguration, input string, tag string) (string, error) {
func getStringAfterTag(input string, tag string) (string, error) {
str := strings.TrimPrefix(input, tag)
str = strings.TrimSpace(str)

Expand Down
6 changes: 6 additions & 0 deletions pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,12 @@ func InitCliConfig(configAndStacksInfo schema.ConfigAndStacksInfo, processStacks
return cliConfig, err
}

// Process stores config
err = processStoreConfig(&cliConfig)
if err != nil {
return cliConfig, err
}

// Process the base path specified in the Terraform provider (which calls into the atmos code)
// This overrides all other atmos base path configs (`atmos.yaml`, ENV var `ATMOS_BASE_PATH`)
if configAndStacksInfo.AtmosBasePath != "" {
Expand Down
15 changes: 15 additions & 0 deletions pkg/config/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ import (
"strconv"
"strings"

"github.com/charmbracelet/log"
"github.com/cloudposse/atmos/pkg/schema"
"github.com/cloudposse/atmos/pkg/store"
u "github.com/cloudposse/atmos/pkg/utils"
)

Expand Down Expand Up @@ -486,6 +488,19 @@ func processCommandLineArgs(cliConfig *schema.CliConfiguration, configAndStacksI
return nil
}

// processStoreConfig creates a store registry from the provided stores config and assigns it to the cliConfig
func processStoreConfig(cliConfig *schema.CliConfiguration) error {
log.Debug("processStoreConfig", "cliConfig.StoresConfig", fmt.Sprintf("%v", cliConfig.StoresConfig))

storeRegistry, err := store.NewStoreRegistry(&cliConfig.StoresConfig)
if err != nil {
return err
}
cliConfig.Stores = storeRegistry

return nil
}

// GetContextFromVars creates a context object from the provided variables
func GetContextFromVars(vars map[string]any) schema.Context {
var context schema.Context
Expand Down
32 changes: 32 additions & 0 deletions pkg/hooks/hooks.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
package hooks

type HookType string

const (
AfterTerraformApply HookType = "after.terraform.apply"
BeforeTerraformApply HookType = "before.terraform.apply"
AfterTerraformPlan HookType = "after.terraform.plan"
BeforeTerraformPlan HookType = "before.terraform.plan"
)

type Command string

const (
Store Command = "store"
)

// Hook defines the structure of a hook
type Hook struct {
Events []string `yaml:"events"`
Command string `yaml:"command"`

// Dynamic command-specific properties

// store command
Name string `yaml:"name,omitempty"` // for store command
Values map[string]interface{} `yaml:"values,omitempty"` // for store command
}

type Hooks struct {
Hooks map[string]Hook `yaml:"hooks"`
}
56 changes: 33 additions & 23 deletions pkg/schema/schema.go
Original file line number Diff line number Diff line change
@@ -1,32 +1,40 @@
package schema

import "github.com/cloudposse/atmos/pkg/store"

type AtmosSectionMapType = map[string]any

// CliConfiguration structure represents schema for `atmos.yaml` CLI config
type CliConfiguration struct {
BasePath string `yaml:"base_path" json:"base_path" mapstructure:"base_path"`
Components Components `yaml:"components" json:"components" mapstructure:"components"`
Stacks Stacks `yaml:"stacks" json:"stacks" mapstructure:"stacks"`
Workflows Workflows `yaml:"workflows,omitempty" json:"workflows,omitempty" mapstructure:"workflows"`
Logs Logs `yaml:"logs,omitempty" json:"logs,omitempty" mapstructure:"logs"`
Commands []Command `yaml:"commands,omitempty" json:"commands,omitempty" mapstructure:"commands"`
CommandAliases CommandAliases `yaml:"aliases,omitempty" json:"aliases,omitempty" mapstructure:"aliases"`
Integrations Integrations `yaml:"integrations,omitempty" json:"integrations,omitempty" mapstructure:"integrations"`
Schemas Schemas `yaml:"schemas,omitempty" json:"schemas,omitempty" mapstructure:"schemas"`
Templates Templates `yaml:"templates,omitempty" json:"templates,omitempty" mapstructure:"templates"`
Settings CliSettings `yaml:"settings,omitempty" json:"settings,omitempty" mapstructure:"settings"`
Vendor Vendor `yaml:"vendor,omitempty" json:"vendor,omitempty" mapstructure:"vendor"`
Initialized bool `yaml:"initialized" json:"initialized" mapstructure:"initialized"`
StacksBaseAbsolutePath string `yaml:"stacksBaseAbsolutePath,omitempty" json:"stacksBaseAbsolutePath,omitempty" mapstructure:"stacksBaseAbsolutePath"`
IncludeStackAbsolutePaths []string `yaml:"includeStackAbsolutePaths,omitempty" json:"includeStackAbsolutePaths,omitempty" mapstructure:"includeStackAbsolutePaths"`
ExcludeStackAbsolutePaths []string `yaml:"excludeStackAbsolutePaths,omitempty" json:"excludeStackAbsolutePaths,omitempty" mapstructure:"excludeStackAbsolutePaths"`
TerraformDirAbsolutePath string `yaml:"terraformDirAbsolutePath,omitempty" json:"terraformDirAbsolutePath,omitempty" mapstructure:"terraformDirAbsolutePath"`
HelmfileDirAbsolutePath string `yaml:"helmfileDirAbsolutePath,omitempty" json:"helmfileDirAbsolutePath,omitempty" mapstructure:"helmfileDirAbsolutePath"`
StackConfigFilesRelativePaths []string `yaml:"stackConfigFilesRelativePaths,omitempty" json:"stackConfigFilesRelativePaths,omitempty" mapstructure:"stackConfigFilesRelativePaths"`
StackConfigFilesAbsolutePaths []string `yaml:"stackConfigFilesAbsolutePaths,omitempty" json:"stackConfigFilesAbsolutePaths,omitempty" mapstructure:"stackConfigFilesAbsolutePaths"`
StackType string `yaml:"stackType,omitempty" json:"StackType,omitempty" mapstructure:"stackType"`
Default bool `yaml:"default" json:"default" mapstructure:"default"`
Version Version `yaml:"version,omitempty" json:"version,omitempty" mapstructure:"version"`
BasePath string `yaml:"base_path" json:"base_path" mapstructure:"base_path"`
Components Components `yaml:"components" json:"components" mapstructure:"components"`
Stacks Stacks `yaml:"stacks" json:"stacks" mapstructure:"stacks"`
Workflows Workflows `yaml:"workflows,omitempty" json:"workflows,omitempty" mapstructure:"workflows"`
Logs Logs `yaml:"logs,omitempty" json:"logs,omitempty" mapstructure:"logs"`
Commands []Command `yaml:"commands,omitempty" json:"commands,omitempty" mapstructure:"commands"`
CommandAliases CommandAliases `yaml:"aliases,omitempty" json:"aliases,omitempty" mapstructure:"aliases"`
Integrations Integrations `yaml:"integrations,omitempty" json:"integrations,omitempty" mapstructure:"integrations"`
Schemas Schemas `yaml:"schemas,omitempty" json:"schemas,omitempty" mapstructure:"schemas"`
Templates Templates `yaml:"templates,omitempty" json:"templates,omitempty" mapstructure:"templates"`
Settings CliSettings `yaml:"settings,omitempty" json:"settings,omitempty" mapstructure:"settings"`
StoresConfig store.StoresConfig `yaml:"stores,omitempty" json:"stores,omitempty" mapstructure:"stores"`
Vendor Vendor `yaml:"vendor,omitempty" json:"vendor,omitempty" mapstructure:"vendor"`
Initialized bool `yaml:"initialized" json:"initialized" mapstructure:"initialized"`
StacksBaseAbsolutePath string `yaml:"stacksBaseAbsolutePath,omitempty" json:"stacksBaseAbsolutePath,omitempty" mapstructure:"stacksBaseAbsolutePath"`
IncludeStackAbsolutePaths []string `yaml:"includeStackAbsolutePaths,omitempty" json:"includeStackAbsolutePaths,omitempty" mapstructure:"includeStackAbsolutePaths"`
ExcludeStackAbsolutePaths []string `yaml:"excludeStackAbsolutePaths,omitempty" json:"excludeStackAbsolutePaths,omitempty" mapstructure:"excludeStackAbsolutePaths"`
TerraformDirAbsolutePath string `yaml:"terraformDirAbsolutePath,omitempty" json:"terraformDirAbsolutePath,omitempty" mapstructure:"terraformDirAbsolutePath"`
HelmfileDirAbsolutePath string `yaml:"helmfileDirAbsolutePath,omitempty" json:"helmfileDirAbsolutePath,omitempty" mapstructure:"helmfileDirAbsolutePath"`
StackConfigFilesRelativePaths []string `yaml:"stackConfigFilesRelativePaths,omitempty" json:"stackConfigFilesRelativePaths,omitempty" mapstructure:"stackConfigFilesRelativePaths"`
StackConfigFilesAbsolutePaths []string `yaml:"stackConfigFilesAbsolutePaths,omitempty" json:"stackConfigFilesAbsolutePaths,omitempty" mapstructure:"stackConfigFilesAbsolutePaths"`
StackType string `yaml:"stackType,omitempty" json:"StackType,omitempty" mapstructure:"stackType"`
Default bool `yaml:"default" json:"default" mapstructure:"default"`
Version Version `yaml:"version,omitempty" json:"version,omitempty" mapstructure:"version"`

// Stores is never read from yaml, it is populated in processStoreConfig and it's used to pass to the populated store
// registry through to the yaml parsing functions when !store is run and to pass the registry to the hooks
// functions to be able to call stores from within hooks.
Stores store.StoreRegistry `yaml:"stores_registry,omitempty" json:"stores_registry,omitempty" mapstructure:"stores_registry"`
}

type CliSettings struct {
Expand Down Expand Up @@ -193,6 +201,7 @@ type ConfigAndStacksInfo struct {
ComponentSettingsSection AtmosSectionMapType
ComponentOverridesSection AtmosSectionMapType
ComponentProvidersSection AtmosSectionMapType
ComponentHooksSection AtmosSectionMapType
ComponentEnvSection AtmosSectionMapType
ComponentEnvList []string
ComponentBackendSection AtmosSectionMapType
Expand Down Expand Up @@ -476,6 +485,7 @@ type BaseComponentConfig struct {
BaseComponentSettings AtmosSectionMapType
BaseComponentEnv AtmosSectionMapType
BaseComponentProviders AtmosSectionMapType
BaseComponentHooks AtmosSectionMapType
FinalBaseComponentName string
BaseComponentCommand string
BaseComponentBackendType string
Expand Down
Loading

0 comments on commit b75be94

Please sign in to comment.