Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Atmos manifest lists merge strategies #609

Merged
merged 11 commits into from
May 27, 2024
Merged

Add Atmos manifest lists merge strategies #609

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
8994bab
updates
aknysh May 26, 2024
8a3cb96
updates
aknysh May 26, 2024
a1fccfb
updates
aknysh May 26, 2024
576d58c
updates
aknysh May 26, 2024
63ad16e
updates
aknysh May 26, 2024
81016ff
updates
aknysh May 26, 2024
3aaf0d2
updates
aknysh May 26, 2024
0a5fa83
updates
aknysh May 26, 2024
9f513c7
updates
aknysh May 26, 2024
8d15c7c
updates
aknysh May 26, 2024
291bd5d
updates
aknysh May 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 17 additions & 4 deletions atmos.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ stacks:
name_pattern: "{tenant}-{environment}-{stage}"

workflows:
# Can also be set using 'ATMOS_WORKFLOWS_BASE_PATH' ENV var, or '--workflows-dir' command-line arguments
# Can also be set using 'ATMOS_WORKFLOWS_BASE_PATH' ENV var, or '--workflows-dir' command-line argument
# Supports both absolute and relative paths
base_path: "stacks/workflows"

Expand Down Expand Up @@ -301,12 +301,12 @@ integrations:
schemas:
# https://json-schema.org
jsonschema:
# Can also be set using 'ATMOS_SCHEMAS_JSONSCHEMA_BASE_PATH' ENV var, or '--schemas-jsonschema-dir' command-line arguments
# Can also be set using 'ATMOS_SCHEMAS_JSONSCHEMA_BASE_PATH' ENV var, or '--schemas-jsonschema-dir' command-line argument
# Supports both absolute and relative paths
base_path: "stacks/schemas/jsonschema"
# https://www.openpolicyagent.org
opa:
# Can also be set using 'ATMOS_SCHEMAS_OPA_BASE_PATH' ENV var, or '--schemas-opa-dir' command-line arguments
# Can also be set using 'ATMOS_SCHEMAS_OPA_BASE_PATH' ENV var, or '--schemas-opa-dir' command-line argument
# Supports both absolute and relative paths
base_path: "stacks/schemas/opa"
# JSON Schema to validate Atmos manifests
Expand All @@ -318,7 +318,7 @@ schemas:
# https://www.schemastore.org/json
# https://github.com/SchemaStore/schemastore
atmos:
# Can also be set using 'ATMOS_SCHEMAS_ATMOS_MANIFEST' ENV var, or '--schemas-atmos-manifest' command-line arguments
# Can also be set using 'ATMOS_SCHEMAS_ATMOS_MANIFEST' ENV var, or '--schemas-atmos-manifest' command-line argument
# Supports both absolute and relative paths (relative to the `base_path` setting in `atmos.yaml`)
manifest: "stacks/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json"

Expand All @@ -338,3 +338,16 @@ templates:
timeout: 5
# https://docs.gomplate.ca/datasources
datasources: {}

settings:
# `list_merge_strategy` specifies how lists are merged in Atmos stack manifests.
# Can also be set using 'ATMOS_SETTINGS_LIST_MERGE_STRATEGY' environment variable, or '--settings-list-merge-strategy' command-line argument
# The following strategies are supported:
# `replace`: Most recent list imported wins (the default behavior).
# `append`: The sequence of lists is appended in the same order as imports.
# `merge`: The items in the destination list are deep-merged with the items in the source list.
# The items in the source list take precedence.
# The items are processed starting from the first up to the length of the source list (the remaining items are not processed).
# If the source and destination lists have the same length, all items in the destination lists are
# deep-merged with all items in the source list.
list_merge_strategy: replace
2 changes: 1 addition & 1 deletion examples/demo-stacks/atmos.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,5 @@ stacks:
name_pattern: "{stage}"

logs:
file: "/dev/stdout"
file: "/dev/stderr"
level: Info
2 changes: 1 addition & 1 deletion examples/quick-start/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ ARG GEODESIC_OS=debian
# https://atmos.tools/
# https://github.com/cloudposse/atmos
# https://github.com/cloudposse/atmos/releases
ARG ATMOS_VERSION=1.75.0
ARG ATMOS_VERSION=1.76.0

# Terraform: https://github.com/hashicorp/terraform/releases
ARG TF_VERSION=1.8.4
Expand Down
21 changes: 17 additions & 4 deletions examples/quick-start/atmos.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ stacks:
name_pattern: "{tenant}-{environment}-{stage}"

workflows:
# Can also be set using 'ATMOS_WORKFLOWS_BASE_PATH' ENV var, or '--workflows-dir' command-line arguments
# Can also be set using 'ATMOS_WORKFLOWS_BASE_PATH' ENV var, or '--workflows-dir' command-line argument
# Supports both absolute and relative paths
base_path: "stacks/workflows"

Expand Down Expand Up @@ -234,12 +234,12 @@ commands:
schemas:
# https://json-schema.org
jsonschema:
# Can also be set using 'ATMOS_SCHEMAS_JSONSCHEMA_BASE_PATH' ENV var, or '--schemas-jsonschema-dir' command-line arguments
# Can also be set using 'ATMOS_SCHEMAS_JSONSCHEMA_BASE_PATH' ENV var, or '--schemas-jsonschema-dir' command-line argument
# Supports both absolute and relative paths
base_path: "stacks/schemas/jsonschema"
# https://www.openpolicyagent.org
opa:
# Can also be set using 'ATMOS_SCHEMAS_OPA_BASE_PATH' ENV var, or '--schemas-opa-dir' command-line arguments
# Can also be set using 'ATMOS_SCHEMAS_OPA_BASE_PATH' ENV var, or '--schemas-opa-dir' command-line argument
# Supports both absolute and relative paths
base_path: "stacks/schemas/opa"
# JSON Schema to validate Atmos manifests
Expand All @@ -251,7 +251,7 @@ schemas:
# https://www.schemastore.org/json
# https://github.com/SchemaStore/schemastore
atmos:
# Can also be set using 'ATMOS_SCHEMAS_ATMOS_MANIFEST' ENV var, or '--schemas-atmos-manifest' command-line arguments
# Can also be set using 'ATMOS_SCHEMAS_ATMOS_MANIFEST' ENV var, or '--schemas-atmos-manifest' command-line argument
# Supports both absolute and relative paths (relative to the `base_path` setting in `atmos.yaml`)
manifest: "stacks/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json"

Expand All @@ -270,3 +270,16 @@ templates:
enabled: true
# https://docs.gomplate.ca/datasources
datasources: {}

settings:
# `list_merge_strategy` specifies how lists are merged in Atmos stack manifests.
# Can also be set using 'ATMOS_SETTINGS_LIST_MERGE_STRATEGY' environment variable, or '--settings-list-merge-strategy' command-line argument
# The following strategies are supported:
# `replace`: Most recent list imported wins (the default behavior).
# `append`: The sequence of lists is appended in the same order as imports.
# `merge`: The items in the destination list are deep-merged with the items in the source list.
# The items in the source list take precedence.
# The items are processed starting from the first up to the length of the source list (the remaining items are not processed).
# If the source and destination lists have the same length, all items in the destination lists are
# deep-merged with all items in the source list.
list_merge_strategy: replace
21 changes: 17 additions & 4 deletions examples/quick-start/rootfs/usr/local/etc/atmos/atmos.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ stacks:
name_pattern: "{tenant}-{environment}-{stage}"

workflows:
# Can also be set using 'ATMOS_WORKFLOWS_BASE_PATH' ENV var, or '--workflows-dir' command-line arguments
# Can also be set using 'ATMOS_WORKFLOWS_BASE_PATH' ENV var, or '--workflows-dir' command-line argument
# Supports both absolute and relative paths
base_path: "stacks/workflows"

Expand Down Expand Up @@ -235,12 +235,12 @@ commands:
schemas:
# https://json-schema.org
jsonschema:
# Can also be set using 'ATMOS_SCHEMAS_JSONSCHEMA_BASE_PATH' ENV var, or '--schemas-jsonschema-dir' command-line arguments
# Can also be set using 'ATMOS_SCHEMAS_JSONSCHEMA_BASE_PATH' ENV var, or '--schemas-jsonschema-dir' command-line argument
# Supports both absolute and relative paths
base_path: "stacks/schemas/jsonschema"
# https://www.openpolicyagent.org
opa:
# Can also be set using 'ATMOS_SCHEMAS_OPA_BASE_PATH' ENV var, or '--schemas-opa-dir' command-line arguments
# Can also be set using 'ATMOS_SCHEMAS_OPA_BASE_PATH' ENV var, or '--schemas-opa-dir' command-line argument
# Supports both absolute and relative paths
base_path: "stacks/schemas/opa"
# JSON Schema to validate Atmos manifests
Expand All @@ -252,7 +252,7 @@ schemas:
# https://www.schemastore.org/json
# https://github.com/SchemaStore/schemastore
atmos:
# Can also be set using 'ATMOS_SCHEMAS_ATMOS_MANIFEST' ENV var, or '--schemas-atmos-manifest' command-line arguments
# Can also be set using 'ATMOS_SCHEMAS_ATMOS_MANIFEST' ENV var, or '--schemas-atmos-manifest' command-line argument
# Supports both absolute and relative paths (relative to the `base_path` setting in `atmos.yaml`)
manifest: "stacks/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json"

Expand All @@ -270,3 +270,16 @@ templates:
enabled: true
# https://docs.gomplate.ca/datasources
datasources: {}

settings:
# `list_merge_strategy` specifies how lists are merged in Atmos stack manifests.
# Can also be set using 'ATMOS_SETTINGS_LIST_MERGE_STRATEGY' environment variable, or '--settings-list-merge-strategy' command-line argument
# The following strategies are supported:
# `replace`: Most recent list imported wins (the default behavior).
# `append`: The sequence of lists is appended in the same order as imports.
# `merge`: The items in the destination list are deep-merged with the items in the source list.
# The items in the source list take precedence.
# The items are processed starting from the first up to the length of the source list (the remaining items are not processed).
# If the source and destination lists have the same length, all items in the destination lists are
# deep-merged with all items in the source list.
list_merge_strategy: replace
21 changes: 17 additions & 4 deletions examples/tests/atmos.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ stacks:
name_template: "{{.vars.tenant}}-{{.vars.environment}}-{{.vars.stage}}"

workflows:
# Can also be set using 'ATMOS_WORKFLOWS_BASE_PATH' ENV var, or '--workflows-dir' command-line arguments
# Can also be set using 'ATMOS_WORKFLOWS_BASE_PATH' ENV var, or '--workflows-dir' command-line argument
# Supports both absolute and relative paths
base_path: "stacks/workflows"

Expand Down Expand Up @@ -328,12 +328,12 @@ integrations:
schemas:
# https://json-schema.org
jsonschema:
# Can also be set using 'ATMOS_SCHEMAS_JSONSCHEMA_BASE_PATH' ENV var, or '--schemas-jsonschema-dir' command-line arguments
# Can also be set using 'ATMOS_SCHEMAS_JSONSCHEMA_BASE_PATH' ENV var, or '--schemas-jsonschema-dir' command-line argument
# Supports both absolute and relative paths
base_path: "stacks/schemas/jsonschema"
# https://www.openpolicyagent.org
opa:
# Can also be set using 'ATMOS_SCHEMAS_OPA_BASE_PATH' ENV var, or '--schemas-opa-dir' command-line arguments
# Can also be set using 'ATMOS_SCHEMAS_OPA_BASE_PATH' ENV var, or '--schemas-opa-dir' command-line argument
# Supports both absolute and relative paths
base_path: "stacks/schemas/opa"
# JSON Schema to validate Atmos manifests
Expand All @@ -345,7 +345,7 @@ schemas:
# https://www.schemastore.org/json
# https://github.com/SchemaStore/schemastore
atmos:
# Can also be set using 'ATMOS_SCHEMAS_ATMOS_MANIFEST' ENV var, or '--schemas-atmos-manifest' command-line arguments
# Can also be set using 'ATMOS_SCHEMAS_ATMOS_MANIFEST' ENV var, or '--schemas-atmos-manifest' command-line argument
# Supports both absolute and relative paths (relative to the `base_path` setting in `atmos.yaml`)
manifest: "../quick-start/stacks/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json"

Expand Down Expand Up @@ -378,3 +378,16 @@ templates:
timeout: 5
# https://docs.gomplate.ca/datasources
datasources: {}

settings:
# `list_merge_strategy` specifies how lists are merged in Atmos stack manifests.
# Can also be set using 'ATMOS_SETTINGS_LIST_MERGE_STRATEGY' environment variable, or '--settings-list-merge-strategy' command-line argument
# The following strategies are supported:
# `replace`: Most recent list imported wins (the default behavior).
# `append`: The sequence of lists is appended in the same order as imports.
# `merge`: The items in the destination list are deep-merged with the items in the source list.
# The items in the source list take precedence.
# The items are processed starting from the first up to the length of the source list (the remaining items are not processed).
# If the source and destination lists have the same length, all items in the destination lists are
# deep-merged with all items in the source list.
list_merge_strategy: replace
21 changes: 17 additions & 4 deletions examples/tests/rootfs/usr/local/etc/atmos/atmos.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ stacks:
name_template: "{{.vars.tenant}}-{{.vars.environment}}-{{.vars.stage}}"

workflows:
# Can also be set using 'ATMOS_WORKFLOWS_BASE_PATH' ENV var, or '--workflows-dir' command-line arguments
# Can also be set using 'ATMOS_WORKFLOWS_BASE_PATH' ENV var, or '--workflows-dir' command-line argument
# Supports both absolute and relative paths
base_path: "stacks/workflows"

Expand Down Expand Up @@ -576,12 +576,12 @@ integrations:
schemas:
# https://json-schema.org
jsonschema:
# Can also be set using 'ATMOS_SCHEMAS_JSONSCHEMA_BASE_PATH' ENV var, or '--schemas-jsonschema-dir' command-line arguments
# Can also be set using 'ATMOS_SCHEMAS_JSONSCHEMA_BASE_PATH' ENV var, or '--schemas-jsonschema-dir' command-line argument
# Supports both absolute and relative paths
base_path: "stacks/schemas/jsonschema"
# https://www.openpolicyagent.org
opa:
# Can also be set using 'ATMOS_SCHEMAS_OPA_BASE_PATH' ENV var, or '--schemas-opa-dir' command-line arguments
# Can also be set using 'ATMOS_SCHEMAS_OPA_BASE_PATH' ENV var, or '--schemas-opa-dir' command-line argument
# Supports both absolute and relative paths
base_path: "stacks/schemas/opa"
# JSON Schema to validate Atmos manifests
Expand All @@ -593,7 +593,7 @@ schemas:
# https://www.schemastore.org/json
# https://github.com/SchemaStore/schemastore
atmos:
# Can also be set using 'ATMOS_SCHEMAS_ATMOS_MANIFEST' ENV var, or '--schemas-atmos-manifest' command-line arguments
# Can also be set using 'ATMOS_SCHEMAS_ATMOS_MANIFEST' ENV var, or '--schemas-atmos-manifest' command-line argument
# Supports both absolute and relative paths (relative to the `base_path` setting in `atmos.yaml`)
manifest: "stacks/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json"

Expand Down Expand Up @@ -625,3 +625,16 @@ templates:
timeout: 5
# https://docs.gomplate.ca/datasources
datasources: {}

settings:
# `list_merge_strategy` specifies how lists are merged in Atmos stack manifests.
# Can also be set using 'ATMOS_SETTINGS_LIST_MERGE_STRATEGY' environment variable, or '--settings-list-merge-strategy' command-line argument
# The following strategies are supported:
# `replace`: Most recent list imported wins (the default behavior).
# `append`: The sequence of lists is appended in the same order as imports.
# `merge`: The items in the destination list are deep-merged with the items in the source list.
# The items in the source list take precedence.
# The items are processed starting from the first up to the length of the source list (the remaining items are not processed).
# If the source and destination lists have the same length, all items in the destination lists are
# deep-merged with all items in the source list.
list_merge_strategy: replace
4 changes: 2 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/cloudposse/atmos
go 1.21

require (
dario.cat/mergo v1.0.0
github.com/Masterminds/sprig/v3 v3.2.3
github.com/alecthomas/chroma v0.10.0
github.com/arsham/figurine v1.3.0
Expand All @@ -21,7 +22,6 @@ require (
github.com/hashicorp/hcl v1.0.0
github.com/hashicorp/hcl/v2 v2.20.1
github.com/hashicorp/terraform-config-inspect v0.0.0-20240509232506-4708120f8f30
github.com/imdario/mergo v0.3.13
github.com/ivanpirog/coloredcobra v1.0.1
github.com/json-iterator/go v1.1.12
github.com/jwalton/go-supportscolor v1.2.0
Expand All @@ -48,7 +48,6 @@ require (
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v1.1.6 // indirect
cloud.google.com/go/storage v1.36.0 // indirect
dario.cat/mergo v1.0.0 // indirect
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver/v3 v3.2.0 // indirect
Expand Down Expand Up @@ -155,6 +154,7 @@ require (
github.com/hashicorp/vault/sdk v0.5.0 // indirect
github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 // indirect
github.com/huandu/xstrings v1.3.3 // indirect
github.com/imdario/mergo v0.3.13 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
Expand Down
14 changes: 14 additions & 0 deletions internal/exec/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,7 @@ func processCommandLineArgs(
configAndStacksInfo.RedirectStdErr = argsAndFlagsInfo.RedirectStdErr
configAndStacksInfo.LogsLevel = argsAndFlagsInfo.LogsLevel
configAndStacksInfo.LogsFile = argsAndFlagsInfo.LogsFile
configAndStacksInfo.SettingsListMergeStrategy = argsAndFlagsInfo.SettingsListMergeStrategy

// Check if `-h` or `--help` flags are specified
if argsAndFlagsInfo.NeedHelp {
Expand Down Expand Up @@ -885,6 +886,19 @@ func processArgsAndFlags(componentType string, inputArgsAndFlags []string) (sche
info.LogsFile = logsFileFlagParts[1]
}

if arg == cfg.SettingsListMergeStrategyFlag {
if len(inputArgsAndFlags) <= (i + 1) {
return info, fmt.Errorf("invalid flag: %s", arg)
}
info.SettingsListMergeStrategy = inputArgsAndFlags[i+1]
} else if strings.HasPrefix(arg+"=", cfg.SettingsListMergeStrategyFlag) {
var settingsListMergeStrategyParts = strings.Split(arg, "=")
if len(settingsListMergeStrategyParts) != 2 {
return info, fmt.Errorf("invalid flag: %s", arg)
}
info.SettingsListMergeStrategy = settingsListMergeStrategyParts[1]
}

if arg == cfg.FromPlanFlag {
info.UseTerraformPlan = true
}
Expand Down
2 changes: 2 additions & 0 deletions pkg/config/const.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,4 +58,6 @@ const (

LogsLevelFlag = "--logs-level"
LogsFileFlag = "--logs-file"

SettingsListMergeStrategyFlag = "--settings-list-merge-strategy"
)
10 changes: 10 additions & 0 deletions pkg/config/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,12 @@ func processEnvVars(cliConfig *schema.CliConfiguration) error {
cliConfig.Logs.Level = logsLevel
}

listMergeStrategy := os.Getenv("ATMOS_SETTINGS_LIST_MERGE_STRATEGY")
if len(listMergeStrategy) > 0 {
u.LogTrace(*cliConfig, fmt.Sprintf("Found ENV var ATMOS_SETTINGS_LIST_MERGE_STRATEGY=%s", listMergeStrategy))
cliConfig.Settings.ListMergeStrategy = listMergeStrategy
}

return nil
}

Expand Down Expand Up @@ -416,6 +422,10 @@ func processCommandLineArgs(cliConfig *schema.CliConfiguration, configAndStacksI
cliConfig.Logs.File = configAndStacksInfo.LogsFile
u.LogTrace(*cliConfig, fmt.Sprintf("Using command line argument '%s=%s'", LogsFileFlag, configAndStacksInfo.LogsFile))
}
if len(configAndStacksInfo.SettingsListMergeStrategy) > 0 {
cliConfig.Settings.ListMergeStrategy = configAndStacksInfo.SettingsListMergeStrategy
u.LogTrace(*cliConfig, fmt.Sprintf("Using command line argument '%s=%s'", SettingsListMergeStrategyFlag, configAndStacksInfo.SettingsListMergeStrategy))
}

return nil
}
Expand Down
Loading
Loading