Reconcile log groups

.gitignore

@@ -12,3 +12,6 @@ ci/credentials.yml
 
 pipeline-*.yml
 aws-broker_manifest.yml
+
+go.work
+go.work.sum

cmd/tasks/elasticache.go → cmd/tasks/elasticache/tags.go

@@ -1,4 +1,4 @@
-package main
+package elasticache
 
 import (
 	"fmt"
@@ -13,6 +13,8 @@ import (
 	brokertags "github.com/cloud-gov/go-broker-tags"
 	"github.com/jinzhu/gorm"
 	"golang.org/x/exp/slices"
+
+	"github.com/cloud-gov/aws-broker/cmd/tasks/tags"
 )
 
 func getElasticacheInstanceArn(elasticacheClient elasticacheiface.ElastiCacheAPI, redisInstance redis.RedisInstance) (string, error) {
@@ -85,7 +87,7 @@ func processElasticacheResource(elasticacheClient elasticacheiface.ElastiCacheAP
 	return nil
 }
 
-func reconcileElasticacheResourceTags(catalog *catalog.Catalog, db *gorm.DB, elasticacheClient elasticacheiface.ElastiCacheAPI, tagManager brokertags.TagManager) error {
+func ReconcileElasticacheResourceTags(catalog *catalog.Catalog, db *gorm.DB, elasticacheClient elasticacheiface.ElastiCacheAPI, tagManager brokertags.TagManager) error {
 	rows, err := db.Model(&redis.RedisInstance{}).Rows()
 	if err != nil {
 		return err
@@ -108,7 +110,7 @@ func reconcileElasticacheResourceTags(catalog *catalog.Catalog, db *gorm.DB, ela
 			return fmt.Errorf("error getting plan %s for cluster %s", redisInstance.PlanID, redisInstance.ClusterID)
 		}
 
-		generatedTags, err := generateTags(
+		generatedTags, err := tags.GenerateTags(
 			tagManager,
 			catalog.RedisService.Name,
 			plan.Name,

cmd/tasks/go.mod

@@ -3,9 +3,9 @@ module github.com/cloud-gov/aws-broker/cmd/tasks
 go 1.23.4
 
 require (
-	github.com/18F/aws-broker v0.0.0-20241218185015-41549a5e6c73
+	github.com/18F/aws-broker v0.0.0-20241220194228-cfd14a5f5a8a
 	github.com/aws/aws-sdk-go v1.55.5
-	github.com/cloud-gov/go-broker-tags v0.0.0-20241218195254-24284a5c54ae
+	github.com/cloud-gov/go-broker-tags v0.0.0-20241218215556-c78c3f147c5a
 	github.com/go-test/deep v1.1.0
 	github.com/jinzhu/gorm v1.9.16
 	golang.org/x/exp v0.0.0-20241215155358-4a5509556b9e
@@ -14,7 +14,7 @@ require (
 require (
 	code.cloudfoundry.org/lager v2.0.0+incompatible // indirect
 	filippo.io/edwards25519 v1.1.0 // indirect
-	github.com/cloudfoundry-community/go-cfclient/v3 v3.0.0-alpha.6 // indirect
+	github.com/cloudfoundry/go-cfclient/v3 v3.0.0-alpha.9 // indirect
 	github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-co-op/gocron v1.37.0 // indirect
@@ -31,7 +31,7 @@ require (
 	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/stretchr/testify v1.10.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
-	golang.org/x/net v0.33.0 // indirect
+	golang.org/x/crypto v0.31.0 // indirect
 	golang.org/x/oauth2 v0.24.0 // indirect
 	gopkg.in/go-playground/validator.v8 v8.18.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

cmd/tasks/go.sum

@@ -2,16 +2,16 @@ code.cloudfoundry.org/lager v2.0.0+incompatible h1:WZwDKDB2PLd/oL+USK4b4aEjUymIe
 code.cloudfoundry.org/lager v2.0.0+incompatible/go.mod h1:O2sS7gKP3HM2iemG+EnwvyNQK7pTSC6Foi4QiMp9sSk=
 filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
 filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
-github.com/18F/aws-broker v0.0.0-20241218185015-41549a5e6c73 h1:4iIDjgt7GVhwNeRtCsNlaQLe5X6Pkue5djrFCPAfZkg=
-github.com/18F/aws-broker v0.0.0-20241218185015-41549a5e6c73/go.mod h1:S6ZkzzUqW2orPapsUi70KDaK4sf0nHEpTkwEpGUNP5o=
+github.com/18F/aws-broker v0.0.0-20241220194228-cfd14a5f5a8a h1:7ilo3ZFresHFgB7tJ/R6aoA5Uwda7RLIwZyCwqHoEfg=
+github.com/18F/aws-broker v0.0.0-20241220194228-cfd14a5f5a8a/go.mod h1:zt7dfEjwODnnMzS5OT/t5VMOSCR/7METYxWdmkZIWn0=
 github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
 github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
 github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
-github.com/cloud-gov/go-broker-tags v0.0.0-20241218195254-24284a5c54ae h1:Dd2FV9uZnpJ0NXim1nDo4rUsmFFEnLfsTdLxkQJpavE=
-github.com/cloud-gov/go-broker-tags v0.0.0-20241218195254-24284a5c54ae/go.mod h1:wfr28TbguhrMrMrjtsnISb8ygREtbPjKL9iXVbFAe3g=
-github.com/cloudfoundry-community/go-cfclient/v3 v3.0.0-alpha.6 h1:mF8LXapcJsG+zqNFSlfWssERIuK0Nf0UEAyAR/s0TAI=
-github.com/cloudfoundry-community/go-cfclient/v3 v3.0.0-alpha.6/go.mod h1:3tjqtK8cGhfhGNhDVKLQ7AaTDzP9K7fyfeNtYqmNWWM=
+github.com/cloud-gov/go-broker-tags v0.0.0-20241218215556-c78c3f147c5a h1:Gw+OpWeOS9Ztg44tKjNO3/C4lFpzTWCPq87fx24iOKo=
+github.com/cloud-gov/go-broker-tags v0.0.0-20241218215556-c78c3f147c5a/go.mod h1:cAg7jfurQqVmzJV0/kqvFzgTbUzP5jNH1avJjbXM/e8=
+github.com/cloudfoundry/go-cfclient/v3 v3.0.0-alpha.9 h1:HK3+nJEPgwlhc5H74aw/V4mVowqWaTKGjHONdVQQ2Vw=
+github.com/cloudfoundry/go-cfclient/v3 v3.0.0-alpha.9/go.mod h1:eUjFfpsU3lRv388wKlXMmkQfsJ9pveUHZEia7AoBCPY=
 github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q=
 github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@@ -46,8 +46,8 @@ github.com/jinzhu/gorm v1.9.16/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBef
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
-github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E=
-github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
+github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
@@ -87,6 +87,8 @@ github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=

cmd/tasks/logs/logs.go

@@ -0,0 +1,13 @@
+package logs
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
+	"github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface"
+)
+
+func DescribeLogGroups(logsClient cloudwatchlogsiface.CloudWatchLogsAPI, logGroupNamePrefix string) (*cloudwatchlogs.DescribeLogGroupsOutput, error) {
+	return logsClient.DescribeLogGroups(&cloudwatchlogs.DescribeLogGroupsInput{
+		LogGroupNamePrefix: aws.String(logGroupNamePrefix),
+	})
+}

cmd/tasks/logs/tags.go

@@ -0,0 +1,15 @@
+package logs
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
+	"github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface"
+)
+
+func TagCloudwatchLogGroup(logsClient cloudwatchlogsiface.CloudWatchLogsAPI, logGroupArn string, tags map[string]*string) error {
+	_, err := logsClient.TagResource(&cloudwatchlogs.TagResourceInput{
+		ResourceArn: aws.String(logGroupArn),
+		Tags:        tags,
+	})
+	return err
+}

cmd/tasks/main.go

@@ -9,14 +9,21 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
 	"github.com/aws/aws-sdk-go/service/elasticache"
 	"github.com/aws/aws-sdk-go/service/opensearchservice"
 	awsRds "github.com/aws/aws-sdk-go/service/rds"
 	brokertags "github.com/cloud-gov/go-broker-tags"
 
+	tasksElasticache "github.com/cloud-gov/aws-broker/cmd/tasks/elasticache"
+	tasksOpensearch "github.com/cloud-gov/aws-broker/cmd/tasks/opensearch"
+	"github.com/cloud-gov/aws-broker/cmd/tasks/rds"
+	tasksRds "github.com/cloud-gov/aws-broker/cmd/tasks/rds"
+
 	"github.com/18F/aws-broker/catalog"
 	"github.com/18F/aws-broker/config"
 	"github.com/18F/aws-broker/db"
+
 	"golang.org/x/exp/slices"
 )
 
@@ -36,14 +43,18 @@ func (s *serviceNames) Set(value string) error {
 var servicesToTag serviceNames
 
 func run() error {
-	actionPtr := flag.String("action", "", "Action to take. Accepted options: 'update-tags'")
+	actionPtr := flag.String("action", "", "Action to take. Accepted options: 'reconcile-tags', 'reconcile-log-groups'")
 	flag.Var(&servicesToTag, "service", "Specify AWS service whose instances should have tags updated. Accepted options: 'rds', 'elasticache', 'elasticsearch', 'opensearch'")
 	flag.Parse()
 
 	if *actionPtr == "" {
 		log.Fatal("--action flag is required")
 	}
 
+	if len(servicesToTag) == 0 {
+		return errors.New("--service argument is required. Specify --service multiple times to update tags for multiple services")
+	}
+
 	var settings config.Settings
 
 	// Load settings from environment
@@ -63,11 +74,7 @@ func run() error {
 		return fmt.Errorf("could not initialize session: %s", err)
 	}
 
-	if *actionPtr == "update-tags" {
-		if len(servicesToTag) == 0 {
-			return errors.New("--service argument is required. Specify --service multiple times to update tags for multiple services")
-		}
-
+	if *actionPtr == "reconcile-tags" {
 		tagManager, err := brokertags.NewCFTagManager(
 			"AWS broker",
 			settings.Environment,
@@ -82,23 +89,37 @@ func run() error {
 		path, _ := os.Getwd()
 		c := catalog.InitCatalog(path)
 
+		logsClient := cloudwatchlogs.New(sess)
+
 		if slices.Contains(servicesToTag, "rds") {
 			rdsClient := awsRds.New(sess)
-			err := reconcileRDSResourceTags(c, db, rdsClient, tagManager)
+			err := tasksRds.ReconcileRDSResourceTags(c, db, rdsClient, logsClient, tagManager)
 			if err != nil {
 				return err
 			}
 		}
 		if slices.Contains(servicesToTag, "elasticache") {
 			elasticacheClient := elasticache.New(sess)
-			err := reconcileElasticacheResourceTags(c, db, elasticacheClient, tagManager)
+			err := tasksElasticache.ReconcileElasticacheResourceTags(c, db, elasticacheClient, tagManager)
 			if err != nil {
 				return err
 			}
 		}
 		if slices.Contains(servicesToTag, "elasticsearch") || slices.Contains(servicesToTag, "opensearch") {
 			opensearchClient := opensearchservice.New(sess)
-			err := reconcileOpensearchResourceTags(c, db, opensearchClient, tagManager)
+			err := tasksOpensearch.ReconcileOpensearchResourceTags(c, db, opensearchClient, tagManager)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	if *actionPtr == "reconcile-log-groups" {
+		logsClient := cloudwatchlogs.New(sess)
+
+		if slices.Contains(servicesToTag, "rds") {
+			rdsClient := awsRds.New(sess)
+			err := rds.ReconcileRDSCloudwatchLogGroups(logsClient, rdsClient, settings.DbNamePrefix, db)
 			if err != nil {
 				return err
 			}

cmd/tasks/elasticsearch.go → cmd/tasks/opensearch/tags.go

@@ -1,4 +1,4 @@
-package main
+package opensearch
 
 import (
 	"fmt"
@@ -10,6 +10,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/service/opensearchservice"
 	"github.com/aws/aws-sdk-go/service/opensearchservice/opensearchserviceiface"
+	"github.com/cloud-gov/aws-broker/cmd/tasks/tags"
 	brokertags "github.com/cloud-gov/go-broker-tags"
 	"github.com/jinzhu/gorm"
 	"golang.org/x/exp/slices"
@@ -85,7 +86,7 @@ func processOpensearchResource(opensearchClient opensearchserviceiface.OpenSearc
 	return nil
 }
 
-func reconcileOpensearchResourceTags(catalog *catalog.Catalog, db *gorm.DB, opensearchClient opensearchserviceiface.OpenSearchServiceAPI, tagManager brokertags.TagManager) error {
+func ReconcileOpensearchResourceTags(catalog *catalog.Catalog, db *gorm.DB, opensearchClient opensearchserviceiface.OpenSearchServiceAPI, tagManager brokertags.TagManager) error {
 	rows, err := db.Model(&elasticsearch.ElasticsearchInstance{}).Rows()
 	if err != nil {
 		return err
@@ -108,7 +109,7 @@ func reconcileOpensearchResourceTags(catalog *catalog.Catalog, db *gorm.DB, open
 			return fmt.Errorf("error getting plan %s for domain %s", elasticsearchInstance.PlanID, elasticsearchInstance.Domain)
 		}
 
-		generatedTags, err := generateTags(
+		generatedTags, err := tags.GenerateTags(
 			tagManager,
 			catalog.ElasticsearchService.Name,
 			plan.Name,

cmd/tasks/rds/logs.go

@@ -0,0 +1,79 @@
+package rds
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"strings"
+
+	"github.com/18F/aws-broker/services/rds"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface"
+	awsRds "github.com/aws/aws-sdk-go/service/rds"
+	"github.com/aws/aws-sdk-go/service/rds/rdsiface"
+	"github.com/jinzhu/gorm"
+
+	"github.com/cloud-gov/aws-broker/cmd/tasks/logs"
+)
+
+func getLogGroupPrefix(prefixParts ...string) string {
+	return fmt.Sprintf("/aws/rds/instance/%s", strings.Join(prefixParts, "/"))
+}
+
+func ReconcileRDSCloudwatchLogGroups(logsClient cloudwatchlogsiface.CloudWatchLogsAPI, rdsClient rdsiface.RDSAPI, dbNamePrefix string, db *gorm.DB) error {
+	resp, err := logs.DescribeLogGroups(logsClient, getLogGroupPrefix(dbNamePrefix))
+	if err != nil {
+		return err
+	}
+
+	for _, logGroup := range resp.LogGroups {
+		log.Printf("found group: %s", *logGroup.LogGroupName)
+		res := strings.Split(*logGroup.LogGroupName, "/")
+		if len(res) < 4 {
+			return fmt.Errorf("error parsing log group name %s", *logGroup.LogGroupName)
+		}
+
+		dbName := res[4]
+		if dbName == "" {
+			return fmt.Errorf("could not get database name for log group %s", *logGroup.LogGroupName)
+		}
+		log.Printf("got database name %s from group %s", dbName, *logGroup.LogGroupName)
+
+		var rdsDatabase rds.RDSInstance
+		err := db.Where(&rds.RDSInstance{Database: dbName}).First(&rdsDatabase).Error
+		if err != nil {
+			if errors.Is(err, gorm.ErrRecordNotFound) {
+				log.Printf("could not find database record with name %s, continuing", dbName)
+				continue
+			} else {
+				return err
+			}
+		}
+
+		resp, err := rdsClient.DescribeDBInstances(&awsRds.DescribeDBInstancesInput{
+			DBInstanceIdentifier: aws.String(dbName),
+		})
+		if err != nil {
+			return err
+		}
+
+		instanceInfo := resp.DBInstances[0]
+
+		var enabledGroups []string
+		for _, enabledGroup := range instanceInfo.EnabledCloudwatchLogsExports {
+			enabledGroups = append(enabledGroups, *enabledGroup)
+		}
+
+		log.Printf("database name %s has log groups enabled %v", dbName, enabledGroups)
+
+		rdsDatabase.EnabledCloudwatchLogGroupExports = enabledGroups
+		err = db.Save(&rdsDatabase).Error
+		if err != nil {
+			return err
+		}
+
+		log.Printf("saved enabled log groups for %s", dbName)
+	}
+
+	return nil
+}