#29 - implement support AWS secrets manager for store SOPS Age keys

cmd/cluster.go

@@ -336,6 +336,10 @@ func (cc *ClusterCommands) provisionDestroyTargetCluster() error {
 			if err := cc.createAWSClusterSSHKey(); err != nil {
 				return err
 			}
+
+			if err := cc.createAWSSecrets(); err != nil {
+				return err
+			}
 		case azure_provider.AzureClusterProvider:
 			if err := cc.createAzureSecrets(cc.Conf.AzureConfigure); err != nil {
 				return err

cmd/cluster_capa.go

@@ -3,12 +3,14 @@ package cmd
 import (
 	"os"
 	"path/filepath"
+	"strings"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	v1 "k8s.io/client-go/applyconfigurations/core/v1"
 
 	"rmk/providers/aws_provider"
+	"rmk/util"
 )
 
 const (
@@ -167,9 +169,42 @@ func (cc *ClusterCommands) getAWSClusterContext() ([]byte, error) {
 }
 
 func (cc *ClusterCommands) createAWSClusterSSHKey() error {
-	return aws_provider.NewAwsConfigure(cc.Ctx.Context, cc.Conf.Profile).CreateEC2SSHKey(cc.Conf.Name)
+	return aws_provider.NewAwsConfigure(cc.Ctx.Context, cc.Conf.Profile).CreateAWSEC2SSHKey(cc.Conf.Name)
 }
 
 func (cc *ClusterCommands) deleteAWSClusterSSHKey() error {
-	return aws_provider.NewAwsConfigure(cc.Ctx.Context, cc.Conf.Profile).DeleteEC2SSHKey(cc.Conf.Name)
+	return aws_provider.NewAwsConfigure(cc.Ctx.Context, cc.Conf.Profile).DeleteAWSEC2SSHKey(cc.Conf.Name)
+}
+
+func (cc *ClusterCommands) createAWSSecrets() error {
+	a := aws_provider.NewAwsConfigure(cc.Ctx.Context, cc.Conf.Profile)
+
+	secrets, err := a.GetAWSSecrets(cc.Conf.Tenant)
+	if err != nil {
+		return err
+	}
+
+	if len(secrets) > 0 {
+		return nil
+	}
+
+	walkMatch, err := util.WalkMatch(cc.Conf.SopsAgeKeys, cc.Conf.Tenant+"*"+util.SopsAgeKeyExt)
+	if err != nil {
+		return err
+	}
+
+	for _, val := range walkMatch {
+		file, err := os.ReadFile(val)
+		if err != nil {
+			return err
+		}
+
+		keyName := strings.TrimSuffix(filepath.Base(val), util.SopsAgeKeyExt)
+
+		if err := a.SetAWSSecret(cc.Conf.Tenant, keyName, file); err != nil {
+			return err
+		}
+	}
+
+	return nil
 }

cmd/config.go

@@ -46,6 +46,18 @@ func (c *ConfigCommands) configAws(profile string) error {
 	ac.ConfigSource = strings.Join(ac.AWSSharedConfigFile(profile), "")
 	ac.CredentialsSource = strings.Join(ac.AWSSharedCredentialsFile(profile), "")
 
+	if !util.IsExists(c.Ctx.String("config"), true) &&
+		util.IsExists(ac.ConfigSource, true) &&
+		util.IsExists(ac.CredentialsSource, true) {
+		if err := os.RemoveAll(ac.ConfigSource); err != nil {
+			return err
+		}
+
+		if err := os.RemoveAll(ac.CredentialsSource); err != nil {
+			return err
+		}
+	}
+
 	if util.IsExists(ac.ConfigSource, true) {
 		if err := ac.ReadAWSConfigProfile(); err != nil {
 			return err
@@ -104,7 +116,7 @@ func (c *ConfigCommands) configAwsMFA() error {
 		c.Conf.AwsConfigure.Profile = c.Conf.AWSMFAProfile
 	}
 
-	if err := c.Conf.GetMFADevicesSerialNumbers(); err != nil {
+	if err := c.Conf.GetAWSMFADevicesSerialNumbers(); err != nil {
 		return err
 	}
 
@@ -154,7 +166,7 @@ func (c *ConfigCommands) configAwsMFA() error {
 	}
 
 	if len(c.Conf.MFADeviceSerialNumber) > 0 && currentTime.After(tokenExpiration) {
-		if err := c.Conf.GetMFASessionToken(); err != nil {
+		if err := c.Conf.GetAWSMFASessionToken(); err != nil {
 			return err
 		}
 
@@ -296,6 +308,24 @@ func initAWSProfile(c *cli.Context, conf *config.Config, gitSpec *git_handler.Gi
 		return err
 	}
 
+	secrets, err := aws_provider.NewAwsConfigure(c.Context, conf.Profile).GetAWSSecrets(conf.Tenant)
+	if err != nil {
+		return err
+	}
+
+	if len(secrets) == 0 {
+		zap.S().Warnf("SOPS Age keys contents for tenant %s not found in %s secrets",
+			conf.Tenant, strings.ToUpper(aws_provider.AWSClusterProvider))
+	}
+
+	for key, val := range secrets {
+		zap.S().Infof("download AWS secret %s to %s",
+			key, filepath.Join(conf.SopsAgeKeys, key+util.SopsAgeKeyExt))
+		if err := os.WriteFile(filepath.Join(conf.SopsAgeKeys, key+util.SopsAgeKeyExt), val, 0644); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -368,6 +398,11 @@ func initAzureProfile(c *cli.Context, conf *config.Config, gitSpec *git_handler.
 			return err
 		}
 
+		if len(secrets) == 0 {
+			zap.S().Warnf("SOPS Age keys contents for tenant %s not found in %s key vault secrets",
+				conf.Tenant, strings.ToUpper(aws_provider.AWSClusterProvider))
+		}
+
 		for key, val := range secrets {
 			zap.S().Infof("download Azure key vault secret %s to %s",
 				key, filepath.Join(conf.SopsAgeKeys, key+util.SopsAgeKeyExt))
@@ -411,6 +446,11 @@ func initGCPProfile(c *cli.Context, conf *config.Config, gitSpec *git_handler.Gi
 		return err
 	}
 
+	if len(secrets) == 0 {
+		zap.S().Warnf("SOPS Age keys contents for tenant %s not found in %s secrets",
+			conf.Tenant, strings.ToUpper(aws_provider.AWSClusterProvider))
+	}
+
 	for key, val := range secrets {
 		zap.S().Infof("download GCP secret %s to %s",
 			key, filepath.Join(conf.SopsAgeKeys, key+util.SopsAgeKeyExt))

cmd/rmk.go

@@ -65,7 +65,7 @@ func docGenerateAction() cli.ActionFunc {
 func getRMKArtifactMetadata(keyPath string) (*RMKArtifactMetadata, error) {
 	rmkArtifactMetadata := &RMKArtifactMetadata{}
 	aws := &aws_provider.AwsConfigure{Region: util.RMKBucketRegion}
-	data, err := aws.GetFileData(util.RMKBucketName, util.RMKBin+"/"+keyPath+"/metadata.json")
+	data, err := aws.GetAWSBucketFileData(util.RMKBucketName, util.RMKBin+"/"+keyPath+"/metadata.json")
 	if err != nil {
 		return nil, err
 	}

cmd/secret.go

@@ -156,6 +156,24 @@ func (sc *SecretCommands) CreateKeys() error {
 func (sc *SecretCommands) DownloadKeys() error {
 	switch sc.Conf.ClusterProvider {
 	case aws_provider.AWSClusterProvider:
+		secrets, err := aws_provider.NewAwsConfigure(sc.Ctx.Context, sc.Conf.Profile).GetAWSSecrets(sc.Conf.Tenant)
+		if err != nil {
+			return err
+		}
+
+		if len(secrets) == 0 {
+			zap.S().Warnf("SOPS Age keys contents for tenant %s not found in %s secrets",
+				sc.Conf.Tenant, strings.ToUpper(aws_provider.AWSClusterProvider))
+		}
+
+		for key, val := range secrets {
+			zap.S().Infof("download AWS secret %s to %s",
+				key, filepath.Join(sc.Conf.SopsAgeKeys, key+util.SopsAgeKeyExt))
+			if err := os.WriteFile(filepath.Join(sc.Conf.SopsAgeKeys, key+util.SopsAgeKeyExt), val, 0644); err != nil {
+				return err
+			}
+		}
+
 		return nil
 	case azure_provider.AzureClusterProvider:
 		if err := sc.Conf.NewAzureClient(sc.Ctx.Context, sc.Conf.Name); err != nil {
@@ -167,6 +185,11 @@ func (sc *SecretCommands) DownloadKeys() error {
 			return err
 		}
 
+		if len(secrets) == 0 {
+			zap.S().Warnf("SOPS Age keys contents for tenant %s not found in %s secrets",
+				sc.Conf.Tenant, strings.ToUpper(aws_provider.AWSClusterProvider))
+		}
+
 		for key, val := range secrets {
 			zap.S().Infof("download Azure key vault secret %s to %s",
 				key, filepath.Join(sc.Conf.SopsAgeKeys, key+util.SopsAgeKeyExt))
@@ -184,6 +207,11 @@ func (sc *SecretCommands) DownloadKeys() error {
 			return err
 		}
 
+		if len(secrets) == 0 {
+			zap.S().Warnf("SOPS Age keys contents for tenant %s not found in %s secrets",
+				sc.Conf.Tenant, strings.ToUpper(aws_provider.AWSClusterProvider))
+		}
+
 		for key, val := range secrets {
 			zap.S().Infof("download GCP secret %s to %s",
 				key, filepath.Join(sc.Conf.SopsAgeKeys, key+util.SopsAgeKeyExt))
@@ -201,6 +229,26 @@ func (sc *SecretCommands) DownloadKeys() error {
 func (sc *SecretCommands) UploadKeys() error {
 	switch sc.Conf.ClusterProvider {
 	case aws_provider.AWSClusterProvider:
+		a := aws_provider.NewAwsConfigure(sc.Ctx.Context, sc.Conf.Profile)
+
+		walkMatch, err := util.WalkMatch(sc.Conf.SopsAgeKeys, sc.Conf.Tenant+"*"+util.SopsAgeKeyExt)
+		if err != nil {
+			return err
+		}
+
+		for _, val := range walkMatch {
+			file, err := os.ReadFile(val)
+			if err != nil {
+				return err
+			}
+
+			keyName := strings.TrimSuffix(filepath.Base(val), util.SopsAgeKeyExt)
+
+			if err := a.SetAWSSecret(sc.Conf.Tenant, keyName, file); err != nil {
+				return err
+			}
+		}
+
 		return nil
 	case azure_provider.AzureClusterProvider:
 		if err := sc.Conf.NewAzureClient(sc.Ctx.Context, sc.Conf.Name); err != nil {

go.mod

@@ -15,21 +15,22 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0
 	github.com/Masterminds/semver v1.5.0
 	github.com/Masterminds/sprig/v3 v3.2.3
-	github.com/aws/aws-sdk-go-v2 v1.32.3
+	github.com/aws/aws-sdk-go-v2 v1.32.6
 	github.com/aws/aws-sdk-go-v2/config v1.26.3
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.11
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.187.0
 	github.com/aws/aws-sdk-go-v2/service/eks v1.51.1
 	github.com/aws/aws-sdk-go-v2/service/iam v1.28.7
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.48.0
+	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.7
 	github.com/aws/aws-sdk-go-v2/service/sts v1.26.7
-	github.com/aws/smithy-go v1.22.0
+	github.com/aws/smithy-go v1.22.1
 	github.com/cheggaaa/pb v1.0.29
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-git/go-git/v5 v5.11.0
 	github.com/google/go-github v17.0.0+incompatible
 	github.com/googleapis/gax-go/v2 v2.12.3
-	github.com/hashicorp/go-getter v1.7.3
+	github.com/hashicorp/go-getter v1.7.5
 	github.com/slack-go/slack v0.12.3
 	github.com/urfave/cli/v2 v2.27.1
 	go.uber.org/zap v1.27.0
@@ -63,8 +64,8 @@ require (
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.16.14 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect

go.sum

@@ -243,8 +243,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/aws/aws-sdk-go v1.44.122 h1:p6mw01WBaNpbdP2xrisz5tIkcNwzj/HysobNoaAHjgo=
 github.com/aws/aws-sdk-go v1.44.122/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
-github.com/aws/aws-sdk-go-v2 v1.32.3 h1:T0dRlFBKcdaUPGNtkBSwHZxrtis8CQU17UpNBZYd0wk=
-github.com/aws/aws-sdk-go-v2 v1.32.3/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo=
+github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG8b4=
+github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo=
 github.com/aws/aws-sdk-go-v2/config v1.26.3 h1:dKuc2jdp10y13dEEvPqWxqLoc0vF3Z9FC45MvuQSxOA=
@@ -255,10 +255,10 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tC
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.11 h1:I6lAa3wBWfCz/cKkOpAcumsETRkFAl70sWi8ItcMEsM=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.11/go.mod h1:be1NIO30kJA23ORBLqPo1LttEM6tPNSEcjkd1eKzNW0=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 h1:Jw50LwEkVjuVzE1NzkhNKkBf9cRN7MtE1F/b2cOKTUM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22/go.mod h1:Y/SmAyPcOTmpeVaWSzSKiILfXTVJwrGmYZhcRbhWuEY=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 h1:981MHwBaRZM7+9QSR6XamDzF/o7ouUGxFzr+nVSIhrs=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22/go.mod h1:1RA1+aBEfn+CAB/Mh0MB6LsdCYCnjZm7tKXtnk499ZQ=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 h1:s/fF4+yDQDoElYhfIVvSNyeCydfbuTKzhxSXDXCPasU=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25/go.mod h1:IgPfDv5jqFIzQSNbUEMoitNooSMXjRSDkhXv8jiROvU=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 h1:ZntTCl5EsYnhN/IygQEUugpdwbhdkom9uHcbCftiGgA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25/go.mod h1:DBdPrgeocww+CSl1C8cEV8PN1mHMBhuCDLpXezyvWkE=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10 h1:5oE2WzJE56/mVveuDZPJESKlg/00AaS2pY2QZcnxg4M=
@@ -279,14 +279,16 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10 h1:KOxnQeWy5sXyS
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10/go.mod h1:jMx5INQFYFYB3lQD9W0D8Ohgq6Wnl7NYOJ2TQndbulI=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.48.0 h1:PJTdBMsyvra6FtED7JZtDpQrIAflYDHFoZAu/sKYkwU=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.48.0/go.mod h1:4qXHrG1Ne3VGIMZPCB8OjH/pLFO94sKABIusjh0KWPU=
+github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.7 h1:Nyfbgei75bohfmZNxgN27i528dGYVzqWJGlAO6lzXy8=
+github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.7/go.mod h1:FG4p/DciRxPgjA+BEOlwRHN0iA8hX2h9g5buSy3cTDA=
 github.com/aws/aws-sdk-go-v2/service/sso v1.18.6 h1:dGrs+Q/WzhsiUKh82SfTVN66QzyulXuMDTV/G8ZxOac=
 github.com/aws/aws-sdk-go-v2/service/sso v1.18.6/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6 h1:Yf2MIo9x+0tyv76GljxzqA3WtC5mw7NmazD2chwjxE4=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8=
 github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0=
 github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U=
-github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM=
-github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
+github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro=
+github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
 github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas=
 github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4=
 github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
@@ -497,8 +499,8 @@ github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
-github.com/hashicorp/go-getter v1.7.3 h1:bN2+Fw9XPFvOCjB0UOevFIMICZ7G2XSQHzfvLUyOM5E=
-github.com/hashicorp/go-getter v1.7.3/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744=
+github.com/hashicorp/go-getter v1.7.5 h1:dT58k9hQ/vbxNMwoI5+xFYAJuv6152UNvdHokfI5wE4=
+github.com/hashicorp/go-getter v1.7.5/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744=
 github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo=
 github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I=
 github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=