From 2e2e8e422a8cbca91e1bd76a100552597d97b5c6 Mon Sep 17 00:00:00 2001 From: German Lashevich Date: Mon, 7 Aug 2023 22:25:50 +0200 Subject: [PATCH 01/19] chore: use const for a constant --- internal/myks/globe.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/myks/globe.go b/internal/myks/globe.go index 40a5d456..ae73f9fa 100644 --- a/internal/myks/globe.go +++ b/internal/myks/globe.go @@ -26,7 +26,7 @@ var prototypesFs embed.FS //go:embed all:assets/envs var environmentsFs embed.FS -var GlobalLogFormat = "\033[1m[global]\033[0m %s" +const GlobalLogFormat = "\033[1m[global]\033[0m %s" // Define the main structure type Globe struct { From b376a62dd75bd8a65cda9372afdcd4e3359c6601 Mon Sep 17 00:00:00 2001 From: German Lashevich Date: Mon, 7 Aug 2023 22:31:34 +0200 Subject: [PATCH 02/19] chore: remove argocd-apps prototype and supporting code --- internal/myks/assets/data-schema.ytt.yaml | 26 --------- .../assets/envs/mykso/dev/env-data.ytt.yaml | 1 - .../prototypes/argocd-apps/app-data.ytt.yaml | 10 ---- .../prototypes/argocd-apps/ytt/all.ytt.yaml | 57 ------------------- internal/myks/globe.go | 36 ------------ 5 files changed, 130 deletions(-) delete mode 100644 internal/myks/assets/prototypes/argocd-apps/app-data.ytt.yaml delete mode 100644 internal/myks/assets/prototypes/argocd-apps/ytt/all.ytt.yaml diff --git a/internal/myks/assets/data-schema.ytt.yaml b/internal/myks/assets/data-schema.ytt.yaml index 84e9c676..d7041f94 100644 --- a/internal/myks/assets/data-schema.ytt.yaml +++ b/internal/myks/assets/data-schema.ytt.yaml @@ -31,29 +31,3 @@ helm: #! Used by myks. #! If defined, passed as a value of `--namespace` for `helm template`. namespace: "" -#! Myks configuration and runtime data. -myks: - applicationDataFileName: "" - applicationNames: [""] - dataSchemaFileName: "" - environmentBaseDir: "" - environmentDataFileName: "" - gitRepoBranch: "" - gitRepoUrl: "" - helmChartsDirName: "" - myksDataFileName: "" - namespacePrefix: "" - prototypesDir: "" - renderedDir: "" - renderedEnvironmentDataFileName: "" - rootDir: "" - searchPaths: [""] - serviceDirName: "" - tempDirName: "" - vendirConfigFileName: "" - vendirLockFileName: "" - vendirSyncFileName: "" - vendorDirName: "" - yttLibraryDirName: "" - yttPkgStepDirName: "" - yttStepDirName: "" diff --git a/internal/myks/assets/envs/mykso/dev/env-data.ytt.yaml b/internal/myks/assets/envs/mykso/dev/env-data.ytt.yaml index c28b6f7d..e345e856 100644 --- a/internal/myks/assets/envs/mykso/dev/env-data.ytt.yaml +++ b/internal/myks/assets/envs/mykso/dev/env-data.ytt.yaml @@ -4,5 +4,4 @@ environment: id: mykso-dev applications: - proto: argocd - - proto: argocd-apps - proto: httpbingo diff --git a/internal/myks/assets/prototypes/argocd-apps/app-data.ytt.yaml b/internal/myks/assets/prototypes/argocd-apps/app-data.ytt.yaml deleted file mode 100644 index b497d2a8..00000000 --- a/internal/myks/assets/prototypes/argocd-apps/app-data.ytt.yaml +++ /dev/null @@ -1,10 +0,0 @@ -#@data/values-schema ---- -#@overlay/match-child-defaults missing_ok=True -application: - destination: - #! Destination is a Kubernetes cluster where the application will be deployed. - #! The value depends on how the cluster is registered in ArgoCD. - name: in-cluster - #! Namespace monitored by ArgoCD. - namespace: argocd diff --git a/internal/myks/assets/prototypes/argocd-apps/ytt/all.ytt.yaml b/internal/myks/assets/prototypes/argocd-apps/ytt/all.ytt.yaml deleted file mode 100644 index 3ee35976..00000000 --- a/internal/myks/assets/prototypes/argocd-apps/ytt/all.ytt.yaml +++ /dev/null @@ -1,57 +0,0 @@ -#@ load("@ytt:data", "data") - -#@ appData = data.values.application -#@ envData = data.values.environment -#@ myks = data.values.myks - -#@ projName = "env-" + envData.id ---- -apiVersion: argoproj.io/v1alpha1 -kind: AppProject -metadata: - name: #@ projName - namespace: #@ appData.namespace -spec: - description: #@ "Project for \"{}\" environment".format(envData.id) - clusterResourceWhitelist: - - group: "*" - kind: "*" - destinations: - - name: #@ appData.destination.name - namespace: "*" - namespaceResourceWhitelist: - - group: "*" - kind: "*" - sourceRepos: - - "*" - -#@ for app in envData.applications: -#@ name = app.name or app.proto ---- -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: #@ "app-{}-{}".format(envData.id, name) - namespace: #@ appData.namespace -spec: - destination: - name: #@ appData.destination.name - namespace: #@ name - project: #@ projName - source: - #! path: kubernetes/rendered/envs/phoenix-platform-staging/grafana - #! TODO: Include myks.rootDir or alike in path. - #! It is not done right now because it is not clear how to ensure that - #! the path is relative to the repository root. - path: #@ "{}/{}/{}".format(myks.renderedDir, envData.id, app.name) - plugin: - name: argocd-vault-plugin-v1.0.0 - repoURL: #@ myks.gitRepoUrl - targetRevision: #@ myks.gitRepoBranch - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true -#@ end diff --git a/internal/myks/globe.go b/internal/myks/globe.go index ae73f9fa..73fb5f8c 100644 --- a/internal/myks/globe.go +++ b/internal/myks/globe.go @@ -1,7 +1,6 @@ package myks import ( - "bytes" "embed" "fmt" "io/fs" @@ -11,7 +10,6 @@ import ( "github.com/creasty/defaults" "github.com/rs/zerolog/log" - yaml "gopkg.in/yaml.v3" ) //go:embed assets/data-schema.ytt.yaml @@ -136,12 +134,6 @@ func (g *Globe) Init(searchPaths []string, applicationNames []string) error { } g.extraYttPaths = append(g.extraYttPaths, dataSchemaFileName) - if configFileName, err := g.dumpConfigAsYaml(); err != nil { - log.Warn().Err(err).Msg("Unable to dump config as yaml") - } else { - g.extraYttPaths = append(g.extraYttPaths, configFileName) - } - g.collectEnvironments(searchPaths) return processItemsInParallel(g.environments, func(item interface{}) error { @@ -203,34 +195,6 @@ func (g *Globe) Bootstrap() error { return nil } -// dumpConfigAsYaml dumps the globe config as yaml to a file and returns the file name -func (g *Globe) dumpConfigAsYaml() (string, error) { - configData := struct { - Myks *Globe `yaml:"myks"` - }{ - Myks: g, - } - var yamlData bytes.Buffer - enc := yaml.NewEncoder(&yamlData) - enc.SetIndent(2) - if err := enc.Encode(configData); err != nil { - return "", err - } - yttData := fmt.Sprintf("#@data/values\n---\n%s", yamlData.String()) - - configFileName := filepath.Join(g.RootDir, g.ServiceDirName, g.TempDirName, g.MyksDataFileName) - if err := os.MkdirAll(filepath.Dir(configFileName), 0o750); err != nil { - return "", err - } - if err := os.WriteFile(configFileName, []byte(yttData), 0o600); err != nil { - return "", err - } - - log.Trace().Str("config file", configFileName).Str("content", yttData).Msg("Dumped config as yaml") - - return configFileName, nil -} - func (g *Globe) createBaseFileStructure() error { envDir := filepath.Join(g.RootDir, g.EnvironmentBaseDir) protoDir := filepath.Join(g.RootDir, g.PrototypesDir) From 8eb5830862a3f9ce8211022caf259e5c88c08be1 Mon Sep 17 00:00:00 2001 From: German Lashevich Date: Mon, 7 Aug 2023 22:35:14 +0200 Subject: [PATCH 03/19] chore: formatting --- cmd/render.go | 1 - internal/myks/application.go | 6 ++++-- internal/myks/template.go | 3 ++- main.go | 3 ++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/cmd/render.go b/cmd/render.go index ac99af07..daf38841 100644 --- a/cmd/render.go +++ b/cmd/render.go @@ -13,7 +13,6 @@ func init() { Short: "Render manifests", Long: "Render manifests", Run: func(cmd *cobra.Command, args []string) { - g := myks.New(".") if err := g.Init(targetEnvironments, targetApplications); err != nil { diff --git a/internal/myks/application.go b/internal/myks/application.go index 057a9577..496c6b0c 100644 --- a/internal/myks/application.go +++ b/internal/myks/application.go @@ -3,11 +3,12 @@ package myks import ( "errors" "fmt" - "github.com/rs/zerolog/log" - yaml "gopkg.in/yaml.v3" "io" "os" "path/filepath" + + "github.com/rs/zerolog/log" + yaml "gopkg.in/yaml.v3" ) const ( @@ -41,6 +42,7 @@ type HelmConfig struct { } var ErrNoVendirConfig = errors.New("no vendir config found") + var ApplicationLogFormat = "\033[1m[%s > %s > %s]\033[0m %s" func NewApplication(e *Environment, name string, prototypeName string) (*Application, error) { diff --git a/internal/myks/template.go b/internal/myks/template.go index 747c10c5..bd520c91 100644 --- a/internal/myks/template.go +++ b/internal/myks/template.go @@ -2,9 +2,10 @@ package myks import ( "embed" - "github.com/rs/zerolog/log" "os" "path/filepath" + + "github.com/rs/zerolog/log" ) //go:embed all:templates/*.yaml diff --git a/main.go b/main.go index 428fba02..dfb6cde6 100644 --- a/main.go +++ b/main.go @@ -1,9 +1,10 @@ package main import ( + "os" + "github.com/rs/zerolog" "github.com/rs/zerolog/log" - "os" "github.com/mykso/myks/cmd" ) From c5ae09fb8b8ec5dde65e755a40894c373e61ab88 Mon Sep 17 00:00:00 2001 From: German Lashevich Date: Mon, 7 Aug 2023 23:06:50 +0200 Subject: [PATCH 04/19] refactor: rename vendir secret template --- internal/myks/template.go | 2 +- internal/myks/templates/{secret.yaml => vendir_secret.ytt.yaml} | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) rename internal/myks/templates/{secret.yaml => vendir_secret.ytt.yaml} (97%) diff --git a/internal/myks/template.go b/internal/myks/template.go index bd520c91..0d2064e2 100644 --- a/internal/myks/template.go +++ b/internal/myks/template.go @@ -16,7 +16,7 @@ func writeSecretFile(secretName string, secretFilePath string, username string, if err != nil { return err } - res, err := runYttWithFilesAndStdin([]string{filepath.Join(os.TempDir(), "templates", "secret.yaml")}, nil, func(name string, args []string) { + res, err := runYttWithFilesAndStdin([]string{filepath.Join(os.TempDir(), "templates", "vendir_secret.ytt.yaml")}, nil, func(name string, args []string) { log.Debug().Msg(msgRunCmd("render vendir secret yaml", name, args)) }, "--data-value=secret_name="+secretName, "--data-value=username="+username, "--data-value=password="+password) if err != nil { diff --git a/internal/myks/templates/secret.yaml b/internal/myks/templates/vendir_secret.ytt.yaml similarity index 97% rename from internal/myks/templates/secret.yaml rename to internal/myks/templates/vendir_secret.ytt.yaml index fbf1fdeb..d530a58e 100644 --- a/internal/myks/templates/secret.yaml +++ b/internal/myks/templates/vendir_secret.ytt.yaml @@ -1,5 +1,7 @@ #@ load("@ytt:data", "data") #@ load("@ytt:base64", "base64") + +--- apiVersion: v1 kind: Secret metadata: From af8c8ea7d5ce072cc7471fcfbb4d646f8b6d283e Mon Sep 17 00:00:00 2001 From: German Lashevich Date: Mon, 7 Aug 2023 23:45:05 +0200 Subject: [PATCH 05/19] refactor: extract logic into getDestinationPath func --- internal/myks/render.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/myks/render.go b/internal/myks/render.go index fc92ff71..cf583b4c 100644 --- a/internal/myks/render.go +++ b/internal/myks/render.go @@ -65,7 +65,7 @@ func (a *Application) runSliceFormatStore(previousStepFile string) error { return err } - destinationDir := filepath.Join(a.e.g.RootDir, a.e.g.RenderedDir, "envs", a.e.Id, a.Name) + destinationDir := a.getDestinationDir() // Cleanup the destination directory before writing new files err = os.RemoveAll(destinationDir) @@ -128,6 +128,10 @@ func (a *Application) storeStepResult(output string, stepName string, stepNumber return file, a.writeTempFile(fileName, output) } +func (a *Application) getDestinationDir() string { + return filepath.Join(a.e.g.RootDir, a.e.g.RenderedDir, "envs", a.e.Id, a.Name) +} + // Generates a file name for each document using kind and name if available func genRenderedResourceFileName(resource map[string]interface{}) string { kind := "NO_KIND" From e8a241c54819b1a9153dfbfec88b99ce29da305f Mon Sep 17 00:00:00 2001 From: German Lashevich Date: Tue, 8 Aug 2023 00:59:36 +0200 Subject: [PATCH 06/19] feat: render ArgoCD application yaml file --- internal/myks/application_plugin_argocd.go | 81 +++++++++++++++++++ internal/myks/assets/data-schema.ytt.yaml | 24 ++++++ internal/myks/environment.go | 11 ++- internal/myks/environment_plugin_argocd.go | 1 + .../templates/argocd_application.ytt.yaml | 31 +++++++ 5 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 internal/myks/application_plugin_argocd.go create mode 100644 internal/myks/environment_plugin_argocd.go create mode 100644 internal/myks/templates/argocd_application.ytt.yaml diff --git a/internal/myks/application_plugin_argocd.go b/internal/myks/application_plugin_argocd.go new file mode 100644 index 00000000..6b59305f --- /dev/null +++ b/internal/myks/application_plugin_argocd.go @@ -0,0 +1,81 @@ +package myks + +import ( + "bytes" + _ "embed" + "path/filepath" + "text/template" +) + +//go:embed templates/argocd_application.ytt.yaml +var argocd_application_template []byte + +const argocd_data_values_schema = ` +#@data/values-schema +--- +argocd: + app: + name: "{{ .AppName }}" + source: + path: "{{ .AppPath }}" +` + +func (a *Application) renderArgoCD() (err error) { + schemaFile, err := a.argoCDPrepareSchema() + if err != nil { + return + } + + res, err := a.yttS( + "argocd", + "create ArgoCD application yaml", + []string{schemaFile}, + bytes.NewReader(argocd_application_template), + ) + if err != nil { + return + } + + filepath := filepath.Join(a.getArgoCDDestinationDir(), "app-"+a.Name+".yaml") + err = writeFile(filepath, []byte(res.Stdout)) + if err != nil { + return + } + + return +} + +func (a *Application) argoCDPrepareSchema() (filename string, err error) { + const name = "argocd_data_schema.ytt.yaml" + + tmpl, err := template.New(name).Parse(string(argocd_data_values_schema)) + if err != nil { + return + } + + type Data struct { + AppName string + AppPath string + } + + data := Data{ + AppName: a.Name, + AppPath: a.getDestinationDir(), + } + + buf := &bytes.Buffer{} + err = tmpl.Execute(buf, data) + if err != nil { + return + } + + err = a.writeTempFile(name, buf.String()) + + filename = a.expandTempPath(name) + + return +} + +func (a *Application) getArgoCDDestinationDir() string { + return filepath.Join(a.e.g.RootDir, a.e.g.RenderedDir, "argocd", a.e.Id) +} diff --git a/internal/myks/assets/data-schema.ytt.yaml b/internal/myks/assets/data-schema.ytt.yaml index d7041f94..ef670fa8 100644 --- a/internal/myks/assets/data-schema.ytt.yaml +++ b/internal/myks/assets/data-schema.ytt.yaml @@ -8,6 +8,30 @@ --- #! Add here any application-specific data. application: {} +argocd: + #! Namespace of the ArgoCD server. + namespace: argocd + app: + #! If not set, the name of the currently rendered application is used. + name: "" + destination: + #! spec.destination.name of the ArgoCD application. + name: in-cluster + #! spec.destination.server of the ArgoCD application. + #! If set, used instead of spec.destination.name. + server: "" + #! spec.destination.namespace of the ArgoCD application. + #! If not set, defaults to argocd.app.name. + namespace: "" + source: + #! spec.source.path of the ArgoCD application. + #! If not set, defaults to the destination path of the currently rendered application. + #! With the default myks configuration: `rendered/envs//` + path: "" + #! spec.source.repoURL of the ArgoCD application. + repoURL: "" + #! spec.source.targetRevision of the ArgoCD application. + targetRevision: main environment: #! Unique identifier of the environment, required by myks. #@schema/validation min_len=1 diff --git a/internal/myks/environment.go b/internal/myks/environment.go index 92c38d8c..d7a28a6e 100644 --- a/internal/myks/environment.go +++ b/internal/myks/environment.go @@ -101,7 +101,11 @@ func (e *Environment) Render() error { &Ytt{ident: "ytt", app: app, additive: false}, &GlobalYtt{ident: "global-ytt", app: app, additive: false}, } - return app.RenderAndSlice(yamlTemplatingTools) + if err := app.RenderAndSlice(yamlTemplatingTools); err != nil { + return err + } + + return app.renderArgoCD() }) } @@ -120,7 +124,10 @@ func (e *Environment) SyncAndRender() error { &Ytt{ident: "ytt", app: app, additive: false}, &GlobalYtt{ident: "global-ytt", app: app, additive: false}, } - return app.RenderAndSlice(yamlTemplatingTools) + if err := app.RenderAndSlice(yamlTemplatingTools); err != nil { + return err + } + return app.renderArgoCD() }) } diff --git a/internal/myks/environment_plugin_argocd.go b/internal/myks/environment_plugin_argocd.go new file mode 100644 index 00000000..1c911a3d --- /dev/null +++ b/internal/myks/environment_plugin_argocd.go @@ -0,0 +1 @@ +package myks diff --git a/internal/myks/templates/argocd_application.ytt.yaml b/internal/myks/templates/argocd_application.ytt.yaml new file mode 100644 index 00000000..a2f6fd8f --- /dev/null +++ b/internal/myks/templates/argocd_application.ytt.yaml @@ -0,0 +1,31 @@ +#@ load("@ytt:data", "data") + +#@ v = data.values +#@ a = v.argocd + +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: #@ a.app.name + namespace: #@ a.namespace +spec: + project: default + destination: + #@ if a.destination.server: + server: #@ a.destination.server + #@ else: + name: #@ a.destination.name + #@ end + namespace: #@ a.destination.namespace or a.app.name + source: + path: #@ a.source.path + repoURL: #@ a.source.repoURL + targetRevision: #@ a.source.targetRevision + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true From 89dfc1f166b897b472ef8f8885a05d7f3572133e Mon Sep 17 00:00:00 2001 From: German Lashevich Date: Mon, 14 Aug 2023 00:37:07 +0200 Subject: [PATCH 07/19] feat: optional ArgoCD plugin --- internal/myks/application.go | 20 ++++++++++++-------- internal/myks/application_plugin_argocd.go | 4 ++++ internal/myks/assets/data-schema.ytt.yaml | 1 + 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/internal/myks/application.go b/internal/myks/application.go index 506b8b19..02803108 100644 --- a/internal/myks/application.go +++ b/internal/myks/application.go @@ -23,16 +23,15 @@ const ( ) type Application struct { - // Name of the application - Name string - // Application prototype directory + Name string Prototype string - // Environment + e *Environment - // YTT data files - yttDataFiles []string - cached bool - yttPkgDirs []string + + argoCDEnabled bool + cached bool + yttDataFiles []string + yttPkgDirs []string } type HelmConfig struct { @@ -84,6 +83,9 @@ func (a *Application) Init() error { return err } + type ArgoCD struct { + Enabled bool + } type Cache struct { Enabled bool } @@ -93,6 +95,7 @@ func (a *Application) Init() error { var applicationData struct { Application struct { + ArgoCD ArgoCD `yaml:"argocd"` Cache Cache `yaml:"cache"` YttPkg YttPkg `yaml:"yttPkg"` } @@ -102,6 +105,7 @@ func (a *Application) Init() error { if err != nil { return err } + a.argoCDEnabled = applicationData.Application.ArgoCD.Enabled a.cached = applicationData.Application.Cache.Enabled a.yttPkgDirs = applicationData.Application.YttPkg.Dirs diff --git a/internal/myks/application_plugin_argocd.go b/internal/myks/application_plugin_argocd.go index 6b59305f..1e7fa6c9 100644 --- a/internal/myks/application_plugin_argocd.go +++ b/internal/myks/application_plugin_argocd.go @@ -21,6 +21,10 @@ argocd: ` func (a *Application) renderArgoCD() (err error) { + if !a.argoCDEnabled { + return + } + schemaFile, err := a.argoCDPrepareSchema() if err != nil { return diff --git a/internal/myks/assets/data-schema.ytt.yaml b/internal/myks/assets/data-schema.ytt.yaml index 80a5226f..9fb31cdb 100644 --- a/internal/myks/assets/data-schema.ytt.yaml +++ b/internal/myks/assets/data-schema.ytt.yaml @@ -9,6 +9,7 @@ #! Add here any application-specific data. application: {} argocd: + enabled: true #! Namespace of the ArgoCD server. namespace: argocd app: From 04f6956e6017bab4df172c8e5458f154c22399fe Mon Sep 17 00:00:00 2001 From: German Lashevich Date: Mon, 14 Aug 2023 01:26:13 +0200 Subject: [PATCH 08/19] feat: enable support for overlays --- internal/myks/application.go | 8 ++++---- internal/myks/application_plugin_argocd.go | 19 +++++++++++++++++-- .../envs/_env/argocd/labels.overlay.ytt.yaml | 10 ++++++++++ .../_apps/httpbingo/argocd/overlay.ytt.yaml | 9 +++++++++ internal/myks/globe.go | 2 ++ 5 files changed, 42 insertions(+), 6 deletions(-) create mode 100644 internal/myks/assets/envs/_env/argocd/labels.overlay.ytt.yaml create mode 100644 internal/myks/assets/envs/mykso/dev/_apps/httpbingo/argocd/overlay.ytt.yaml diff --git a/internal/myks/application.go b/internal/myks/application.go index 02803108..0fa1d0d3 100644 --- a/internal/myks/application.go +++ b/internal/myks/application.go @@ -84,10 +84,10 @@ func (a *Application) Init() error { } type ArgoCD struct { - Enabled bool + Enabled bool `yaml:"enabled"` } type Cache struct { - Enabled bool + Enabled bool `yaml:"enabled"` } type YttPkg struct { Dirs []string `yaml:"dirs"` @@ -95,17 +95,17 @@ func (a *Application) Init() error { var applicationData struct { Application struct { - ArgoCD ArgoCD `yaml:"argocd"` Cache Cache `yaml:"cache"` YttPkg YttPkg `yaml:"yttPkg"` } + ArgoCD ArgoCD `yaml:"argocd"` } err = yaml.Unmarshal(dataYaml, &applicationData) if err != nil { return err } - a.argoCDEnabled = applicationData.Application.ArgoCD.Enabled + a.argoCDEnabled = applicationData.ArgoCD.Enabled a.cached = applicationData.Application.Cache.Enabled a.yttPkgDirs = applicationData.Application.YttPkg.Dirs diff --git a/internal/myks/application_plugin_argocd.go b/internal/myks/application_plugin_argocd.go index 1e7fa6c9..8909a3fb 100644 --- a/internal/myks/application_plugin_argocd.go +++ b/internal/myks/application_plugin_argocd.go @@ -5,13 +5,17 @@ import ( _ "embed" "path/filepath" "text/template" + + "github.com/rs/zerolog/log" ) +const ArgoCDStepName = "argocd" + //go:embed templates/argocd_application.ytt.yaml var argocd_application_template []byte const argocd_data_values_schema = ` -#@data/values-schema +#@data/values --- argocd: app: @@ -22,6 +26,7 @@ argocd: func (a *Application) renderArgoCD() (err error) { if !a.argoCDEnabled { + log.Debug().Msg(a.Msg(ArgoCDStepName, "ArgoCD is disabled")) return } @@ -30,10 +35,20 @@ func (a *Application) renderArgoCD() (err error) { return } + // 0. Global data values schema and library files are added later in the a.yttS call + // 1. Dynamyc ArgoCD data values + yttFiles := []string{schemaFile} + // 2. Collection of application-specific data values and schemas + yttFiles = append(yttFiles, a.yttDataFiles...) + // 3. Collection of environment-specific data values and schemas, and overlays + yttFiles = append(yttFiles, a.e.collectBySubpath(filepath.Join("_env", a.e.g.ArgoCDDataDirName))...) + // 4. Collection of application-specific data values and schemas, and overlays + yttFiles = append(yttFiles, a.e.collectBySubpath(filepath.Join("_apps", a.Name, a.e.g.ArgoCDDataDirName))...) + res, err := a.yttS( "argocd", "create ArgoCD application yaml", - []string{schemaFile}, + yttFiles, bytes.NewReader(argocd_application_template), ) if err != nil { diff --git a/internal/myks/assets/envs/_env/argocd/labels.overlay.ytt.yaml b/internal/myks/assets/envs/_env/argocd/labels.overlay.ytt.yaml new file mode 100644 index 00000000..d54e92d8 --- /dev/null +++ b/internal/myks/assets/envs/_env/argocd/labels.overlay.ytt.yaml @@ -0,0 +1,10 @@ +#@ load("@ytt:overlay", "overlay") +#@ load("@ytt:data", "data") + +#@overlay/match by=overlay.subset({"kind": "Application"}) +--- +#@overlay/match missing_ok=True +#@overlay/match-child-defaults missing_ok=True +metadata: + annotations: + myks.dev/environment: #@ data.values.environment.id diff --git a/internal/myks/assets/envs/mykso/dev/_apps/httpbingo/argocd/overlay.ytt.yaml b/internal/myks/assets/envs/mykso/dev/_apps/httpbingo/argocd/overlay.ytt.yaml new file mode 100644 index 00000000..d2eb8e51 --- /dev/null +++ b/internal/myks/assets/envs/mykso/dev/_apps/httpbingo/argocd/overlay.ytt.yaml @@ -0,0 +1,9 @@ +#@ load("@ytt:overlay", "overlay") + +#@overlay/match by=overlay.subset({"kind": "Application"}) +--- +spec: + syncPolicy: + automated: + #! Disable self-healing of the application to allow manual changes. + selfHeal: false diff --git a/internal/myks/globe.go b/internal/myks/globe.go index d7252e8d..5bd418cf 100644 --- a/internal/myks/globe.go +++ b/internal/myks/globe.go @@ -47,6 +47,8 @@ type Globe struct { // Application data file name ApplicationDataFileName string `default:"app-data.ytt.yaml" yaml:"applicationDataFileName"` + // ArgoCD data directory name + ArgoCDDataDirName string `default:"argocd" yaml:"argoCDDataDirName"` // Data values schema file name DataSchemaFileName string `default:"data-schema.ytt.yaml" yaml:"dataSchemaFileName"` // Environment data file name From 293561b80266d36508cdbd8f23adfadb4d4c8a30 Mon Sep 17 00:00:00 2001 From: German Lashevich Date: Mon, 14 Aug 2023 01:40:17 +0200 Subject: [PATCH 09/19] refactor: move argocd template files --- internal/myks/application_plugin_argocd.go | 2 +- .../application.ytt.yaml} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename internal/myks/templates/{argocd_application.ytt.yaml => argocd/application.ytt.yaml} (100%) diff --git a/internal/myks/application_plugin_argocd.go b/internal/myks/application_plugin_argocd.go index 8909a3fb..09425ebf 100644 --- a/internal/myks/application_plugin_argocd.go +++ b/internal/myks/application_plugin_argocd.go @@ -11,7 +11,7 @@ import ( const ArgoCDStepName = "argocd" -//go:embed templates/argocd_application.ytt.yaml +//go:embed templates/argocd/application.ytt.yaml var argocd_application_template []byte const argocd_data_values_schema = ` diff --git a/internal/myks/templates/argocd_application.ytt.yaml b/internal/myks/templates/argocd/application.ytt.yaml similarity index 100% rename from internal/myks/templates/argocd_application.ytt.yaml rename to internal/myks/templates/argocd/application.ytt.yaml From 5a340da1830083117312d93e9677af4549afb0f6 Mon Sep 17 00:00:00 2001 From: German Lashevich Date: Mon, 14 Aug 2023 02:11:20 +0200 Subject: [PATCH 10/19] feat: argocd project rendered for environment --- internal/myks/application_plugin_argocd.go | 6 +-- .../envs/_env/argocd/labels.overlay.ytt.yaml | 4 +- internal/myks/environment.go | 12 +++++ internal/myks/environment_plugin_argocd.go | 46 +++++++++++++++++++ .../myks/templates/argocd/appproject.ytt.yaml | 24 ++++++++++ 5 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 internal/myks/templates/argocd/appproject.ytt.yaml diff --git a/internal/myks/application_plugin_argocd.go b/internal/myks/application_plugin_argocd.go index 09425ebf..8338fe04 100644 --- a/internal/myks/application_plugin_argocd.go +++ b/internal/myks/application_plugin_argocd.go @@ -38,11 +38,11 @@ func (a *Application) renderArgoCD() (err error) { // 0. Global data values schema and library files are added later in the a.yttS call // 1. Dynamyc ArgoCD data values yttFiles := []string{schemaFile} - // 2. Collection of application-specific data values and schemas + // 2. Collection of application main data values and schemas yttFiles = append(yttFiles, a.yttDataFiles...) - // 3. Collection of environment-specific data values and schemas, and overlays + // 3. Collection of environment argocd-specific data values and schemas, and overlays yttFiles = append(yttFiles, a.e.collectBySubpath(filepath.Join("_env", a.e.g.ArgoCDDataDirName))...) - // 4. Collection of application-specific data values and schemas, and overlays + // 4. Collection of application argocd-specific data values and schemas, and overlays yttFiles = append(yttFiles, a.e.collectBySubpath(filepath.Join("_apps", a.Name, a.e.g.ArgoCDDataDirName))...) res, err := a.yttS( diff --git a/internal/myks/assets/envs/_env/argocd/labels.overlay.ytt.yaml b/internal/myks/assets/envs/_env/argocd/labels.overlay.ytt.yaml index d54e92d8..1fb298ea 100644 --- a/internal/myks/assets/envs/_env/argocd/labels.overlay.ytt.yaml +++ b/internal/myks/assets/envs/_env/argocd/labels.overlay.ytt.yaml @@ -1,10 +1,12 @@ #@ load("@ytt:overlay", "overlay") #@ load("@ytt:data", "data") -#@overlay/match by=overlay.subset({"kind": "Application"}) +#! Match all ArgoCD resources: Application, AppProject, Secret. +#@overlay/match by=overlay.all, expects="1+" --- #@overlay/match missing_ok=True #@overlay/match-child-defaults missing_ok=True metadata: annotations: myks.dev/environment: #@ data.values.environment.id + app.kubernetes.io/source: #@ data.values.argocd.source.repoURL diff --git a/internal/myks/environment.go b/internal/myks/environment.go index d7a28a6e..d277ed79 100644 --- a/internal/myks/environment.go +++ b/internal/myks/environment.go @@ -36,6 +36,7 @@ type Environment struct { // Globe instance g *Globe + argoCDEnabled bool // Runtime data renderedEnvDataFilePath string // Found applications @@ -90,6 +91,9 @@ func (e *Environment) Sync() error { } func (e *Environment) Render() error { + if err := e.renderArgoCD(); err != nil { + return err + } return processItemsInParallel(e.Applications, func(item interface{}) error { app, ok := item.(*Application) if !ok { @@ -110,6 +114,9 @@ func (e *Environment) Render() error { } func (e *Environment) SyncAndRender() error { + if err := e.renderArgoCD(); err != nil { + return err + } return processItemsInParallel(e.Applications, func(item interface{}) error { app, ok := item.(*Application) if !ok { @@ -217,6 +224,9 @@ func (e *Environment) saveRenderedEnvData(envDataYaml []byte) error { func (e *Environment) setEnvDataFromYaml(envDataYaml []byte) error { var envDataStruct struct { + ArgoCD struct { + Enabled bool + } Environment struct { Applications []struct { Name string @@ -230,6 +240,8 @@ func (e *Environment) setEnvDataFromYaml(envDataYaml []byte) error { return err } + e.argoCDEnabled = envDataStruct.ArgoCD.Enabled + for _, app := range envDataStruct.Environment.Applications { proto := app.Proto if len(proto) == 0 { diff --git a/internal/myks/environment_plugin_argocd.go b/internal/myks/environment_plugin_argocd.go index 1c911a3d..03a84f71 100644 --- a/internal/myks/environment_plugin_argocd.go +++ b/internal/myks/environment_plugin_argocd.go @@ -1 +1,47 @@ package myks + +import ( + "bytes" + _ "embed" + "path/filepath" + + "github.com/rs/zerolog/log" +) + +//go:embed templates/argocd/appproject.ytt.yaml +var argocd_appproject_template []byte + +func (e *Environment) renderArgoCD() (err error) { + if !e.argoCDEnabled { + log.Debug().Msg(e.Msg("ArgoCD is disabled")) + return + } + + // 0. Global data values schema and library files are added later in the a.yttS call + // 1. Collection of environment main data values and schemas + yttFiles := e.collectBySubpath(e.g.EnvironmentDataFileName) + // 2. Collection of environment argocd-specific data values and schemas, and overlays + yttFiles = append(yttFiles, e.collectBySubpath(filepath.Join("_env", e.g.ArgoCDDataDirName))...) + + res, err := e.yttS( + "create ArgoCD project yaml", + yttFiles, + bytes.NewReader(argocd_appproject_template), + ) + if err != nil { + log.Error().Err(err).Str("stdout", res.Stdout).Str("stderr", res.Stderr).Msg(e.Msg("failed to render ArgoCD project yaml")) + return + } + + filepath := filepath.Join(e.getArgoCDDestinationDir(), "appproject.yaml") + err = writeFile(filepath, []byte(res.Stdout)) + if err != nil { + return + } + + return +} + +func (e *Environment) getArgoCDDestinationDir() string { + return filepath.Join(e.g.RootDir, e.g.RenderedDir, "argocd", e.Id) +} diff --git a/internal/myks/templates/argocd/appproject.ytt.yaml b/internal/myks/templates/argocd/appproject.ytt.yaml new file mode 100644 index 00000000..4e332e13 --- /dev/null +++ b/internal/myks/templates/argocd/appproject.ytt.yaml @@ -0,0 +1,24 @@ +#@ load("@ytt:data", "data") + +#@ a = data.values.argocd +#@ e = data.values.environment + +--- +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: #@ e.id + namespace: #@ a.namespace +spec: + description: #@ 'Project for "{}" environment'.format(e.id) + clusterResourceWhitelist: + - group: "*" + kind: "*" + destinations: + - name: #@ a.destination.name + namespace: "*" + namespaceResourceWhitelist: + - group: "*" + kind: "*" + sourceRepos: + - "*" From d5b82ce53f072e8450a730d6e6cd6636bd9476c2 Mon Sep 17 00:00:00 2001 From: German Lashevich Date: Thu, 17 Aug 2023 01:40:48 +0200 Subject: [PATCH 11/19] feat: better ArgoCD project and secret support --- internal/myks/application_plugin_argocd.go | 4 +- internal/myks/assets/data-schema.ytt.yaml | 53 ++++++++++++------- .../envs/_env/argocd/labels.overlay.ytt.yaml | 2 +- .../envs/_env/argocd/secret.overlay.ytt.yaml | 15 ++++++ internal/myks/environment_plugin_argocd.go | 2 +- .../templates/argocd/application.ytt.yaml | 23 ++++---- .../myks/templates/argocd/appproject.ytt.yaml | 24 --------- .../templates/argocd/environment.ytt.yaml | 41 ++++++++++++++ 8 files changed, 108 insertions(+), 56 deletions(-) create mode 100644 internal/myks/assets/envs/_env/argocd/secret.overlay.ytt.yaml delete mode 100644 internal/myks/templates/argocd/appproject.ytt.yaml create mode 100644 internal/myks/templates/argocd/environment.ytt.yaml diff --git a/internal/myks/application_plugin_argocd.go b/internal/myks/application_plugin_argocd.go index 8338fe04..35e530a5 100644 --- a/internal/myks/application_plugin_argocd.go +++ b/internal/myks/application_plugin_argocd.go @@ -20,8 +20,8 @@ const argocd_data_values_schema = ` argocd: app: name: "{{ .AppName }}" - source: - path: "{{ .AppPath }}" + source: + path: "{{ .AppPath }}" ` func (a *Application) renderArgoCD() (err error) { diff --git a/internal/myks/assets/data-schema.ytt.yaml b/internal/myks/assets/data-schema.ytt.yaml index 9fb31cdb..c5d708a6 100644 --- a/internal/myks/assets/data-schema.ytt.yaml +++ b/internal/myks/assets/data-schema.ytt.yaml @@ -15,24 +15,41 @@ argocd: app: #! If not set, the name of the currently rendered application is used. name: "" - destination: - #! spec.destination.name of the ArgoCD application. - name: in-cluster - #! spec.destination.server of the ArgoCD application. - #! If set, used instead of spec.destination.name. - server: "" - #! spec.destination.namespace of the ArgoCD application. - #! If not set, defaults to argocd.app.name. - namespace: "" - source: - #! spec.source.path of the ArgoCD application. - #! If not set, defaults to the destination path of the currently rendered application. - #! With the default myks configuration: `rendered/envs//` - path: "" - #! spec.source.repoURL of the ArgoCD application. - repoURL: "" - #! spec.source.targetRevision of the ArgoCD application. - targetRevision: main + #! Prefix of the ArgoCD application name. + prefix: "" + destination: + #! spec.destination.name of the ArgoCD application. + name: in-cluster + #! spec.destination.server of the ArgoCD application. + #! If set, used instead of spec.destination.name. + server: "" + #! spec.destination.namespace of the ArgoCD application. + #! If not set, defaults to argocd.app.name. + namespace: "" + source: + #! spec.source.path of the ArgoCD application. + #! If not set, defaults to the destination path of the currently rendered application. + #! With the default myks configuration: `rendered/envs//` + path: "" + #! spec.source.repoURL of the ArgoCD application. + repoURL: "" + #! spec.source.targetRevision of the ArgoCD application. + targetRevision: main + env: + #! If not set, the name of the currently rendered environment is used (environment.id). + name: "" + #! Prefix of a target cluster name. + prefix: "" + #! If set to true, a dummy secret is generated for the target cluster. + #! The user has to create an overlay to set correct values for the secret. + #! See https://argo-cd.readthedocs.io/en/release-2.8/operator-manual/declarative-setup/#clusters + #! TODO: add link to the example overlay. + generateSecret: true + project: + #! If not set, the name of the currently rendered environment is used (environment.id). + name: "" + #! Prefix of the ArgoCD project name. + prefix: "" environment: #! Unique identifier of the environment, required by myks. #@schema/validation min_len=1 diff --git a/internal/myks/assets/envs/_env/argocd/labels.overlay.ytt.yaml b/internal/myks/assets/envs/_env/argocd/labels.overlay.ytt.yaml index 1fb298ea..11a3afb5 100644 --- a/internal/myks/assets/envs/_env/argocd/labels.overlay.ytt.yaml +++ b/internal/myks/assets/envs/_env/argocd/labels.overlay.ytt.yaml @@ -9,4 +9,4 @@ metadata: annotations: myks.dev/environment: #@ data.values.environment.id - app.kubernetes.io/source: #@ data.values.argocd.source.repoURL + app.kubernetes.io/source: #@ data.values.argocd.app.source.repoURL diff --git a/internal/myks/assets/envs/_env/argocd/secret.overlay.ytt.yaml b/internal/myks/assets/envs/_env/argocd/secret.overlay.ytt.yaml new file mode 100644 index 00000000..328e0e80 --- /dev/null +++ b/internal/myks/assets/envs/_env/argocd/secret.overlay.ytt.yaml @@ -0,0 +1,15 @@ +#@ load("@ytt:overlay", "overlay") +--- +#@ def secret_fragment(): +kind: Secret +metadata: + labels: + argocd.argoproj.io/secret-type: cluster +#@ end + +#@overlay/match by=overlay.subset(secret_fragment()), expects="0+" +--- +#! See https://argo-cd.readthedocs.io/en/release-2.8/operator-manual/declarative-setup/#clusters +stringData: + config: ARGOCD_CLUSTER_CONNECT_CONFIG + server: ARGOCD_CLUSTER_SERVER_URL diff --git a/internal/myks/environment_plugin_argocd.go b/internal/myks/environment_plugin_argocd.go index 03a84f71..a204269a 100644 --- a/internal/myks/environment_plugin_argocd.go +++ b/internal/myks/environment_plugin_argocd.go @@ -8,7 +8,7 @@ import ( "github.com/rs/zerolog/log" ) -//go:embed templates/argocd/appproject.ytt.yaml +//go:embed templates/argocd/environment.ytt.yaml var argocd_appproject_template []byte func (e *Environment) renderArgoCD() (err error) { diff --git a/internal/myks/templates/argocd/application.ytt.yaml b/internal/myks/templates/argocd/application.ytt.yaml index a2f6fd8f..d803f0be 100644 --- a/internal/myks/templates/argocd/application.ytt.yaml +++ b/internal/myks/templates/argocd/application.ytt.yaml @@ -1,27 +1,30 @@ #@ load("@ytt:data", "data") -#@ v = data.values -#@ a = v.argocd +#@ a = data.values.argocd +#@ e = data.values.environment + +#@ app_name = a.app.prefix + a.app.name +#@ env_name = a.env.prefix + (a.env.name or e.id) --- apiVersion: argoproj.io/v1alpha1 kind: Application metadata: - name: #@ a.app.name + name: #@ app_name namespace: #@ a.namespace spec: project: default destination: - #@ if a.destination.server: - server: #@ a.destination.server + #@ if a.app.destination.server: + server: #@ a.app.destination.server #@ else: - name: #@ a.destination.name + name: #@ a.app.destination.name or env_name #@ end - namespace: #@ a.destination.namespace or a.app.name + namespace: #@ a.app.destination.namespace or a.app.name source: - path: #@ a.source.path - repoURL: #@ a.source.repoURL - targetRevision: #@ a.source.targetRevision + path: #@ a.app.source.path + repoURL: #@ a.app.source.repoURL + targetRevision: #@ a.app.source.targetRevision syncPolicy: automated: prune: true diff --git a/internal/myks/templates/argocd/appproject.ytt.yaml b/internal/myks/templates/argocd/appproject.ytt.yaml deleted file mode 100644 index 4e332e13..00000000 --- a/internal/myks/templates/argocd/appproject.ytt.yaml +++ /dev/null @@ -1,24 +0,0 @@ -#@ load("@ytt:data", "data") - -#@ a = data.values.argocd -#@ e = data.values.environment - ---- -apiVersion: argoproj.io/v1alpha1 -kind: AppProject -metadata: - name: #@ e.id - namespace: #@ a.namespace -spec: - description: #@ 'Project for "{}" environment'.format(e.id) - clusterResourceWhitelist: - - group: "*" - kind: "*" - destinations: - - name: #@ a.destination.name - namespace: "*" - namespaceResourceWhitelist: - - group: "*" - kind: "*" - sourceRepos: - - "*" diff --git a/internal/myks/templates/argocd/environment.ytt.yaml b/internal/myks/templates/argocd/environment.ytt.yaml new file mode 100644 index 00000000..fd63011d --- /dev/null +++ b/internal/myks/templates/argocd/environment.ytt.yaml @@ -0,0 +1,41 @@ +#@ load("@ytt:data", "data") + +#@ a = data.values.argocd +#@ e = data.values.environment + +#@ env_name = a.env.prefix + (a.env.name or e.id) +#@ project_name = a.project.prefix + (a.env.name or e.id) + +--- +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: #@ project_name + namespace: #@ a.namespace +spec: + description: #@ 'Project for "{}" environment'.format(env_name) + clusterResourceWhitelist: + - group: "*" + kind: "*" + destinations: + - name: #@ env_name + namespace: "*" + namespaceResourceWhitelist: + - group: "*" + kind: "*" + sourceRepos: + - "*" + +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + argocd.argoproj.io/secret-type: cluster + name: #@ env_name + namespace: #@ a.namespace +stringData: + config: ARGOCD_CLUSTER_CONNECT_CONFIG + name: #@ env_name + project: #@ project_name + server: ARGOCD_CLUSTER_SERVER_URL From 7d8e87d7c17c179058063dcae5e2f7c765f38615 Mon Sep 17 00:00:00 2001 From: German Lashevich Date: Thu, 17 Aug 2023 01:52:27 +0200 Subject: [PATCH 12/19] chore: change filename for environment ArgoCD resources --- internal/myks/environment_plugin_argocd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/myks/environment_plugin_argocd.go b/internal/myks/environment_plugin_argocd.go index a204269a..4590c115 100644 --- a/internal/myks/environment_plugin_argocd.go +++ b/internal/myks/environment_plugin_argocd.go @@ -33,7 +33,7 @@ func (e *Environment) renderArgoCD() (err error) { return } - filepath := filepath.Join(e.getArgoCDDestinationDir(), "appproject.yaml") + filepath := filepath.Join(e.getArgoCDDestinationDir(), "env-"+e.Id+".yaml") err = writeFile(filepath, []byte(res.Stdout)) if err != nil { return From 58f45bed4bbe87bae7aaeef7f7adbd2f0632f1fa Mon Sep 17 00:00:00 2001 From: German Lashevich Date: Sat, 19 Aug 2023 01:01:05 +0200 Subject: [PATCH 13/19] chore: remove default value of argocd.destination.name --- internal/myks/assets/data-schema.ytt.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/myks/assets/data-schema.ytt.yaml b/internal/myks/assets/data-schema.ytt.yaml index c5d708a6..06889e2c 100644 --- a/internal/myks/assets/data-schema.ytt.yaml +++ b/internal/myks/assets/data-schema.ytt.yaml @@ -19,7 +19,8 @@ argocd: prefix: "" destination: #! spec.destination.name of the ArgoCD application. - name: in-cluster + #! If not set, defaults to the name of the current environment. + name: "" #! spec.destination.server of the ArgoCD application. #! If set, used instead of spec.destination.name. server: "" From 320fc869071e9c492cc6be1ecb0c9534360a47db Mon Sep 17 00:00:00 2001 From: German Lashevich Date: Sat, 19 Aug 2023 01:07:53 +0200 Subject: [PATCH 14/19] chore: add yaml struct annotations --- internal/myks/environment.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/myks/environment.go b/internal/myks/environment.go index cea556c6..7d17e9f1 100644 --- a/internal/myks/environment.go +++ b/internal/myks/environment.go @@ -147,8 +147,8 @@ func (e *Environment) setId() error { var envData struct { Environment struct { - Id string - } + Id string `yaml:"id"` + } `yaml:"environment"` } err = yaml.Unmarshal(yamlBytes, &envData) if err != nil { @@ -225,14 +225,14 @@ func (e *Environment) saveRenderedEnvData(envDataYaml []byte) error { func (e *Environment) setEnvDataFromYaml(envDataYaml []byte) error { var envDataStruct struct { ArgoCD struct { - Enabled bool - } + Enabled bool `yaml:"enabled"` + } `yaml:"argocd"` Environment struct { Applications []struct { - Name string - Proto string - } - } + Name string `yaml:"name"` + Proto string `yaml:"proto"` + } `yaml:"applications"` + } `yaml:"environment"` } err := yaml.Unmarshal(envDataYaml, &envDataStruct) if err != nil { From 006710363c852f0962d5113aceb2d5ac69f2fb0f Mon Sep 17 00:00:00 2001 From: German Lashevich Date: Sat, 19 Aug 2023 01:08:41 +0200 Subject: [PATCH 15/19] chore: remove unused structures --- internal/myks/environment.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/internal/myks/environment.go b/internal/myks/environment.go index 7d17e9f1..b1448461 100644 --- a/internal/myks/environment.go +++ b/internal/myks/environment.go @@ -14,15 +14,6 @@ import ( var EnvLogFormat = "\033[1m[%s > %s]\033[0m %s" -type ManifestApplication struct { - Name string - Prototype string -} - -type EnvironmentManifest struct { - Applications []ManifestApplication -} - type Environment struct { // Path to the environment directory Dir string From a35c7af87cd579a4832d5dd658e839b5b7a731d0 Mon Sep 17 00:00:00 2001 From: German Lashevich Date: Sat, 19 Aug 2023 01:09:32 +0200 Subject: [PATCH 16/19] Apply suggestions from code review Co-authored-by: Kris Budde --- internal/myks/templates/argocd/application.ytt.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/myks/templates/argocd/application.ytt.yaml b/internal/myks/templates/argocd/application.ytt.yaml index d803f0be..3cbcd889 100644 --- a/internal/myks/templates/argocd/application.ytt.yaml +++ b/internal/myks/templates/argocd/application.ytt.yaml @@ -5,7 +5,8 @@ #@ app_name = a.app.prefix + a.app.name #@ env_name = a.env.prefix + (a.env.name or e.id) - +#@ env_name = a.env.prefix + (a.env.name or e.id) +#@ project_name = a.project.prefix + (a.env.name or e.id) --- apiVersion: argoproj.io/v1alpha1 kind: Application @@ -13,7 +14,7 @@ metadata: name: #@ app_name namespace: #@ a.namespace spec: - project: default + project: #@ project_name destination: #@ if a.app.destination.server: server: #@ a.app.destination.server From f91946747ee8c56c9475099b9b55105f52901185 Mon Sep 17 00:00:00 2001 From: German Lashevich Date: Sat, 19 Aug 2023 01:11:01 +0200 Subject: [PATCH 17/19] chore: fix review suggestions --- internal/myks/templates/argocd/application.ytt.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/myks/templates/argocd/application.ytt.yaml b/internal/myks/templates/argocd/application.ytt.yaml index 3cbcd889..63cd195a 100644 --- a/internal/myks/templates/argocd/application.ytt.yaml +++ b/internal/myks/templates/argocd/application.ytt.yaml @@ -5,8 +5,8 @@ #@ app_name = a.app.prefix + a.app.name #@ env_name = a.env.prefix + (a.env.name or e.id) -#@ env_name = a.env.prefix + (a.env.name or e.id) #@ project_name = a.project.prefix + (a.env.name or e.id) + --- apiVersion: argoproj.io/v1alpha1 kind: Application From b9f6b3f90087db66a108934f86bb4d87adad019e Mon Sep 17 00:00:00 2001 From: German Lashevich Date: Sat, 19 Aug 2023 02:13:40 +0200 Subject: [PATCH 18/19] feat: provide runtime git configuration --- internal/myks/application_plugin_argocd.go | 18 +++- internal/myks/assets/data-schema.ytt.yaml | 12 ++- ....ytt.yaml => annotations.overlay.ytt.yaml} | 2 +- internal/myks/globe.go | 97 ++++++++++++++----- 4 files changed, 97 insertions(+), 32 deletions(-) rename internal/myks/assets/envs/_env/argocd/{labels.overlay.ytt.yaml => annotations.overlay.ytt.yaml} (82%) diff --git a/internal/myks/application_plugin_argocd.go b/internal/myks/application_plugin_argocd.go index 35e530a5..3a21d517 100644 --- a/internal/myks/application_plugin_argocd.go +++ b/internal/myks/application_plugin_argocd.go @@ -22,6 +22,8 @@ argocd: name: "{{ .AppName }}" source: path: "{{ .AppPath }}" + repoURL: "{{ .RepoURL }}" + targetRevision: "{{ .TargetRevision }}" ` func (a *Application) renderArgoCD() (err error) { @@ -52,6 +54,10 @@ func (a *Application) renderArgoCD() (err error) { bytes.NewReader(argocd_application_template), ) if err != nil { + log.Error().Err(err). + Str("stdout", res.Stdout). + Str("stderr", res.Stderr). + Msg(a.Msg("argocd", "failed to render ArgoCD Application yaml")) return } @@ -73,13 +79,17 @@ func (a *Application) argoCDPrepareSchema() (filename string, err error) { } type Data struct { - AppName string - AppPath string + AppName string + AppPath string + RepoURL string + TargetRevision string } data := Data{ - AppName: a.Name, - AppPath: a.getDestinationDir(), + AppName: a.Name, + AppPath: a.getDestinationDir(), + RepoURL: a.e.g.GitRepoUrl, + TargetRevision: a.e.g.GitRepoBranch, } buf := &bytes.Buffer{} diff --git a/internal/myks/assets/data-schema.ytt.yaml b/internal/myks/assets/data-schema.ytt.yaml index 06889e2c..0e5909cb 100644 --- a/internal/myks/assets/data-schema.ytt.yaml +++ b/internal/myks/assets/data-schema.ytt.yaml @@ -33,9 +33,11 @@ argocd: #! With the default myks configuration: `rendered/envs//` path: "" #! spec.source.repoURL of the ArgoCD application. + #! If not set, defaults to the current git repository URL. repoURL: "" #! spec.source.targetRevision of the ArgoCD application. - targetRevision: main + #! If not set, defaults to the current git branch. + targetRevision: "" env: #! If not set, the name of the currently rendered environment is used (environment.id). name: "" @@ -78,3 +80,11 @@ helm: #! Used by myks. #! If defined, passed as a value of `--namespace` for `helm template`. namespace: "" +#! Myks configuration and runtime data. +myks: + #! Set by myks. + #! Set to the current git branch if available. + gitRepoBranch: "" + #! Set by myks. + #! Set to the current git repository URL if available. + gitRepoUrl: "" diff --git a/internal/myks/assets/envs/_env/argocd/labels.overlay.ytt.yaml b/internal/myks/assets/envs/_env/argocd/annotations.overlay.ytt.yaml similarity index 82% rename from internal/myks/assets/envs/_env/argocd/labels.overlay.ytt.yaml rename to internal/myks/assets/envs/_env/argocd/annotations.overlay.ytt.yaml index 11a3afb5..b5d45ef7 100644 --- a/internal/myks/assets/envs/_env/argocd/labels.overlay.ytt.yaml +++ b/internal/myks/assets/envs/_env/argocd/annotations.overlay.ytt.yaml @@ -9,4 +9,4 @@ metadata: annotations: myks.dev/environment: #@ data.values.environment.id - app.kubernetes.io/source: #@ data.values.argocd.app.source.repoURL + app.kubernetes.io/source: #@ data.values.myks.gitRepoUrl diff --git a/internal/myks/globe.go b/internal/myks/globe.go index d859debb..182e8aa6 100644 --- a/internal/myks/globe.go +++ b/internal/myks/globe.go @@ -1,6 +1,7 @@ package myks import ( + "bytes" "embed" "fmt" "io/fs" @@ -10,6 +11,7 @@ import ( "github.com/creasty/defaults" "github.com/rs/zerolog/log" + yaml "gopkg.in/yaml.v3" ) //go:embed assets/data-schema.ytt.yaml @@ -33,64 +35,64 @@ type Globe struct { /// Globe configuration // Base directory for environments - EnvironmentBaseDir string `default:"envs" yaml:"environmentBaseDir"` + EnvironmentBaseDir string `default:"envs"` // Prefix for kubernetes namespaces - NamespacePrefix string `default:"" yaml:"namespacePrefix"` + NamespacePrefix string `default:""` // Application prototypes directory - PrototypesDir string `default:"prototypes" yaml:"prototypesDir"` + PrototypesDir string `default:"prototypes"` // Rendered kubernetes manifests directory - RenderedDir string `default:"rendered" yaml:"renderedDir"` + RenderedDir string `default:"rendered"` // Project root directory - RootDir string `default:"." yaml:"rootDir"` + RootDir string `default:"."` /// Globe constants // Application data file name - ApplicationDataFileName string `default:"app-data.ytt.yaml" yaml:"applicationDataFileName"` + ApplicationDataFileName string `default:"app-data.ytt.yaml"` // ArgoCD data directory name - ArgoCDDataDirName string `default:"argocd" yaml:"argoCDDataDirName"` + ArgoCDDataDirName string `default:"argocd"` // Data values schema file name - DataSchemaFileName string `default:"data-schema.ytt.yaml" yaml:"dataSchemaFileName"` + DataSchemaFileName string `default:"data-schema.ytt.yaml"` // Environment data file name - EnvironmentDataFileName string `default:"env-data.ytt.yaml" yaml:"environmentDataFileName"` + EnvironmentDataFileName string `default:"env-data.ytt.yaml"` // Helm charts directory name - HelmChartsDirName string `default:"charts" yaml:"helmChartsDirName"` - // Myks runtime config file name - MyksDataFileName string `default:"myks-data.ytt.yaml" yaml:"myksDataFileName"` + HelmChartsDirName string `default:"charts"` + // Myks runtime data file name + MyksDataFileName string `default:"myks-data.ytt.yaml"` // Rendered environment data file name - RenderedEnvironmentDataFileName string `default:"env-data.yaml" yaml:"renderedEnvironmentDataFileName"` + RenderedEnvironmentDataFileName string `default:"env-data.yaml"` // Service directory name - ServiceDirName string `default:".myks" yaml:"serviceDirName"` + ServiceDirName string `default:".myks"` // Temporary directory name - TempDirName string `default:"tmp" yaml:"tempDirName"` + TempDirName string `default:"tmp"` // Rendered vendir config file name - VendirConfigFileName string `default:"vendir.yaml" yaml:"vendirConfigFileName"` + VendirConfigFileName string `default:"vendir.yaml"` // Rendered vendir lock file name - VendirLockFileName string `default:"vendir.lock.yaml" yaml:"vendirLockFileName"` + VendirLockFileName string `default:"vendir.lock.yaml"` // Rendered vendir sync file name - VendirSyncFileName string `default:"vendir.sync.yaml" yaml:"vendirSyncFileName"` + VendirSyncFileName string `default:"vendir.sync.yaml"` // Downloaded third-party sources - VendorDirName string `default:"vendor" yaml:"vendorDirName"` + VendorDirName string `default:"vendor"` // Ytt library directory name - YttLibraryDirName string `default:"lib" yaml:"yttLibraryDirName"` + YttLibraryDirName string `default:"lib"` // Ytt step directory name - YttPkgStepDirName string `default:"ytt-pkg" yaml:"yttPkgStepDirName"` + YttPkgStepDirName string `default:"ytt-pkg"` // Ytt step directory name - YttStepDirName string `default:"ytt" yaml:"yttStepDirName"` + YttStepDirName string `default:"ytt"` /// User input // Application names to process - ApplicationNames []string `yaml:"applicationNames"` + ApplicationNames []string // Paths to scan for environments - SearchPaths []string `yaml:"searchPaths"` + SearchPaths []string /// Runtime data // Git repository branch - GitRepoBranch string `yaml:"gitRepoBranch"` + GitRepoBranch string // Git repository URL - GitRepoUrl string `yaml:"gitRepoUrl"` + GitRepoUrl string // Collected environments for processing environments map[string]*Environment @@ -99,6 +101,12 @@ type Globe struct { extraYttPaths []string } +// YttGlobe controls runtime data available to ytt templates +type YttGlobeData struct { + GitRepoBranch string `yaml:"gitRepoBranch"` + GitRepoUrl string `yaml:"gitRepoUrl"` +} + func New(rootDir string) *Globe { g := &Globe{ RootDir: rootDir, @@ -138,6 +146,12 @@ func (g *Globe) Init(asyncLevel int, searchPaths []string, applicationNames []st } g.extraYttPaths = append(g.extraYttPaths, dataSchemaFileName) + if configFileName, err := g.dumpConfigAsYaml(); err != nil { + log.Warn().Err(err).Msg("Unable to dump config as yaml") + } else { + g.extraYttPaths = append(g.extraYttPaths, configFileName) + } + g.collectEnvironments(searchPaths) return process(asyncLevel, g.environments, func(item interface{}) error { @@ -199,6 +213,37 @@ func (g *Globe) Bootstrap(force bool) error { return nil } +// dumpConfigAsYaml dumps the globe config as yaml to a file and returns the file name +func (g *Globe) dumpConfigAsYaml() (string, error) { + configData := struct { + Myks *YttGlobeData `yaml:"myks"` + }{ + Myks: &YttGlobeData{ + GitRepoBranch: g.GitRepoBranch, + GitRepoUrl: g.GitRepoUrl, + }, + } + var yamlData bytes.Buffer + enc := yaml.NewEncoder(&yamlData) + enc.SetIndent(2) + if err := enc.Encode(configData); err != nil { + return "", err + } + yttData := fmt.Sprintf("#@data/values\n---\n%s", yamlData.String()) + + configFileName := filepath.Join(g.RootDir, g.ServiceDirName, g.TempDirName, g.MyksDataFileName) + if err := os.MkdirAll(filepath.Dir(configFileName), 0o750); err != nil { + return "", err + } + if err := os.WriteFile(configFileName, []byte(yttData), 0o600); err != nil { + return "", err + } + + log.Trace().Str("config file", configFileName).Str("content", yttData).Msg("Dumped config as yaml") + + return configFileName, nil +} + func (g *Globe) createBaseFileStructure(force bool) error { envDir := filepath.Join(g.RootDir, g.EnvironmentBaseDir) protoDir := filepath.Join(g.RootDir, g.PrototypesDir) From 2cec0d53fa795c121c28979eb80e55e3b16d4833 Mon Sep 17 00:00:00 2001 From: German Lashevich Date: Sat, 19 Aug 2023 02:14:00 +0200 Subject: [PATCH 19/19] chore: remove unused code --- internal/myks/util.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/internal/myks/util.go b/internal/myks/util.go index 7dd138a8..69162475 100644 --- a/internal/myks/util.go +++ b/internal/myks/util.go @@ -272,10 +272,5 @@ func runYttWithFilesAndStdin(paths []string, stdin io.Reader, log func(name stri } cmdArgs = append(cmdArgs, args...) - res, err := runCmd("ytt", stdin, cmdArgs, log) - if err != nil { - return res, err - } - - return res, err + return runCmd("ytt", stdin, cmdArgs, log) }