From bd8f8ad694edecb290ef8ae39ac5900477c8f6bb Mon Sep 17 00:00:00 2001 From: Renato Rudnicki <77694243+renato-rudnicki@users.noreply.github.com> Date: Tue, 7 May 2024 20:31:43 -0300 Subject: [PATCH] feat: support deletion of Cloud Asset Inventory feeds not in use in organization (#198) Co-authored-by: Daniel Andrade --- modules/project_cleanup/README.md | 3 + .../project_cleanup/function_source/go.mod | 4 + .../project_cleanup/function_source/go.sum | 8 ++ .../project_cleanup/function_source/main.go | 104 +++++++++++++++++- modules/project_cleanup/main.tf | 3 + modules/project_cleanup/variables.tf | 12 ++ 6 files changed, 129 insertions(+), 5 deletions(-) diff --git a/modules/project_cleanup/README.md b/modules/project_cleanup/README.md index 223a4799..de7d905a 100644 --- a/modules/project_cleanup/README.md +++ b/modules/project_cleanup/README.md @@ -16,6 +16,7 @@ The following services must be enabled on the project housing the cleanup functi - Cloud Scheduler (`cloudscheduler.googleapis.com`) - Cloud Resource Manager (`cloudresourcemanager.googleapis.com`) - Compute Engine API (`compute.googleapis.com`) +- Cloud Asset API (`cloudasset.googleapis.com`) - Security Command Center API (`securitycenter.googleapis.com`) @@ -23,6 +24,7 @@ The following services must be enabled on the project housing the cleanup functi | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| clean\_up\_org\_level\_cai\_feeds | Clean up organization level Cloud Asset Inventory Feeds. | `bool` | `false` | no | | clean\_up\_org\_level\_scc\_notifications | Clean up organization level Security Command Center notifications. | `bool` | `false` | no | | clean\_up\_org\_level\_tag\_keys | Clean up organization level Tag Keys. | `bool` | `false` | no | | function\_timeout\_s | The amount of time in seconds allotted for the execution of the function. | `number` | `500` | no | @@ -35,6 +37,7 @@ The following services must be enabled on the project housing the cleanup functi | target\_excluded\_labels | Map of project lablels that won't be deleted. | `map(string)` | `{}` | no | | target\_excluded\_tagkeys | List of organization Tag Key short names that won't be deleted. | `list(string)` | `[]` | no | | target\_folder\_id | Folder ID to delete all projects under. | `string` | `""` | no | +| target\_included\_feeds | List of organization level Cloud Asset Inventory feeds that should be deleted. Regex example: `.*/feeds/fd-cai-monitoring-.*` | `list(string)` | `[]` | no | | target\_included\_labels | Map of project lablels that will be deleted. | `map(string)` | `{}` | no | | target\_included\_scc\_notifications | List of organization Security Command Center notifications names regex that will be deleted. Regex example: `.*/notificationConfigs/scc-notify-.*` | `list(string)` | `[]` | no | | target\_tag\_name | The name of a tag to filter GCP projects on for consideration by the cleanup utility (legacy, use `target_included_labels` map instead). | `string` | `""` | no | diff --git a/modules/project_cleanup/function_source/go.mod b/modules/project_cleanup/function_source/go.mod index 6b06eaca..51aec458 100644 --- a/modules/project_cleanup/function_source/go.mod +++ b/modules/project_cleanup/function_source/go.mod @@ -3,6 +3,7 @@ module github.com/terraform-google-modules/terraform-google-scheduled-function/m go 1.21 require ( + cloud.google.com/go/asset v1.17.2 cloud.google.com/go/securitycenter v1.29.0 golang.org/x/net v0.24.0 golang.org/x/oauth2 v0.20.0 @@ -11,11 +12,14 @@ require ( require ( cloud.google.com/go v0.112.2 // indirect + cloud.google.com/go/accesscontextmanager v1.8.5 // indirect cloud.google.com/go/auth v0.3.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect cloud.google.com/go/compute/metadata v0.3.0 // indirect cloud.google.com/go/iam v1.1.7 // indirect cloud.google.com/go/longrunning v0.5.6 // indirect + cloud.google.com/go/orgpolicy v1.12.1 // indirect + cloud.google.com/go/osconfig v1.12.5 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/modules/project_cleanup/function_source/go.sum b/modules/project_cleanup/function_source/go.sum index 0b80bf12..d0dd2c54 100644 --- a/modules/project_cleanup/function_source/go.sum +++ b/modules/project_cleanup/function_source/go.sum @@ -1,6 +1,10 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.112.2 h1:ZaGT6LiG7dBzi6zNOvVZwacaXlmf3lRqnC4DQzqyRQw= cloud.google.com/go v0.112.2/go.mod h1:iEqjp//KquGIJV/m+Pk3xecgKNhV+ry+vVTsy4TbDms= +cloud.google.com/go/accesscontextmanager v1.8.5 h1:2GLNaNu9KRJhJBFTIVRoPwk6xE5mUDgD47abBq4Zp/I= +cloud.google.com/go/accesscontextmanager v1.8.5/go.mod h1:TInEhcZ7V9jptGNqN3EzZ5XMhT6ijWxTGjzyETwmL0Q= +cloud.google.com/go/asset v1.17.2 h1:xgFnBP3luSbUcC9RWJvb3Zkt+y/wW6PKwPHr3ssnIP8= +cloud.google.com/go/asset v1.17.2/go.mod h1:SVbzde67ehddSoKf5uebOD1sYw8Ab/jD/9EIeWg99q4= cloud.google.com/go/auth v0.3.0 h1:PRyzEpGfx/Z9e8+lHsbkoUVXD0gnu4MNmm7Gp8TQNIs= cloud.google.com/go/auth v0.3.0/go.mod h1:lBv6NKTWp8E3LPzmO1TbiiRKc4drLOfHsgmlH9ogv5w= cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= @@ -11,6 +15,10 @@ cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM= cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA= cloud.google.com/go/longrunning v0.5.6 h1:xAe8+0YaWoCKr9t1+aWe+OeQgN/iJK1fEgZSXmjuEaE= cloud.google.com/go/longrunning v0.5.6/go.mod h1:vUaDrWYOMKRuhiv6JBnn49YxCPz2Ayn9GqyjaBT8/mA= +cloud.google.com/go/orgpolicy v1.12.1 h1:2JbXigqBJVp8Dx5dONUttFqewu4fP0p3pgOdIZAhpYU= +cloud.google.com/go/orgpolicy v1.12.1/go.mod h1:aibX78RDl5pcK3jA8ysDQCFkVxLj3aOQqrbBaUL2V5I= +cloud.google.com/go/osconfig v1.12.5 h1:Mo5jGAxOMKH/PmDY7fgY19yFcVbvwREb5D5zMPQjFfo= +cloud.google.com/go/osconfig v1.12.5/go.mod h1:D9QFdxzfjgw3h/+ZaAb5NypM8bhOMqBzgmbhzWViiW8= cloud.google.com/go/securitycenter v1.29.0 h1:jKqAqF4iLwAzCe9vzncKuxGKStmxX/HL0ItvQJdFqEA= cloud.google.com/go/securitycenter v1.29.0/go.mod h1:1+5P3FIDLvV2lQ83UFn+aQRb5emi4ew2XYrMmd1GSHU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= diff --git a/modules/project_cleanup/function_source/main.go b/modules/project_cleanup/function_source/main.go index 6d312d4f..4ff6f2ce 100644 --- a/modules/project_cleanup/function_source/main.go +++ b/modules/project_cleanup/function_source/main.go @@ -27,6 +27,8 @@ import ( "strings" "time" + asset "cloud.google.com/go/asset/apiv1" + "cloud.google.com/go/asset/apiv1/assetpb" securitycenter "cloud.google.com/go/securitycenter/apiv1" "cloud.google.com/go/securitycenter/apiv1/securitycenterpb" "golang.org/x/net/context" @@ -55,6 +57,8 @@ const ( targetFolderRegexp = `^[0-9]+$` targetOrganizationRegexp = `^[0-9]+$` SCCNotificationsPageSize = "SCC_NOTIFICATIONS_PAGE_SIZE" + CleanUpCaiFeeds = "CLEAN_UP_CAI_FEEDS" + TargetIncludedFeeds = "TARGET_INCLUDED_FEEDS" ) var ( @@ -69,6 +73,8 @@ var ( rootFolderId = getCorrectFolderIdOrTerminateExecution() organizationId = getCorrectOrganizationIdOrTerminateExecution() sccPageSize = getSCCNotificationPageSizeOrTerminateExecution() + cleanUpCaiFeeds = getCleanUpFeedsOrTerminateExecution() + includedFeedsList = getFeedsListFromEnv(TargetIncludedFeeds) ) type PubSubMessage struct { @@ -172,12 +178,12 @@ func checkIfAtLeastOneLabelPresentIfAny(project *cloudresourcemanager.Project, l return result } -func checkIfSCCNotificationNameIncluded(notificationName string, includedSCCNotfis []*regexp.Regexp) bool { - if len(includedSCCNotfis) == 0 { +func checkIfNameIncluded(name string, reg []*regexp.Regexp) bool { + if len(reg) == 0 { return false } - for _, name := range includedSCCNotfis { - if name.MatchString(notificationName) { + for _, regex := range reg { + if regex.MatchString(name) { return true } } @@ -296,6 +302,48 @@ func getSCCNotificationPageSizeOrTerminateExecution() int32 { return int32(size) } +func getFeedsListFromEnv(envVariableName string) []*regexp.Regexp { + var compiledRegEx []*regexp.Regexp + targetIncludedFeeds := os.Getenv(envVariableName) + logger.Println("Try to get CAI Feeds list") + if targetIncludedFeeds == "" { + logger.Printf("No CAI Feeds provided.") + return compiledRegEx + } + + var caiFeeds []string + err := json.Unmarshal([]byte(targetIncludedFeeds), &caiFeeds) + if err != nil { + logger.Printf("Failed to get CAI Feeds list from [%s] env variable, error [%s]", envVariableName, err.Error()) + return nil + } else { + logger.Printf("Got CAI Feeds list [%s] from [%s] env variable", caiFeeds, envVariableName) + } + + //build Regexes + for _, r := range caiFeeds { + result, err := regexp.Compile(r) + if err != nil { + logger.Printf("Invalid regular expression [%s] for CAI Feed", r) + } else { + compiledRegEx = append(compiledRegEx, result) + } + } + return compiledRegEx +} + +func getCleanUpFeedsOrTerminateExecution() bool { + cleanUpCaiFeeds, exists := os.LookupEnv(CleanUpCaiFeeds) + if !exists { + logger.Fatalf("Clean up CAI Feeds environment variable [%s] not set, set the environment variable and try again.", CleanUpCaiFeeds) + } + result, err := strconv.ParseBool(cleanUpCaiFeeds) + if err != nil { + logger.Fatalf("Invalid Clean up CAI Feeds value [%s], specify correct value for environment variable [%s] and try again.", cleanUpCaiFeeds, CleanUpCaiFeeds) + } + return result +} + func getCorrectFolderIdOrTerminateExecution() string { targetFolderIdString := os.Getenv(TargetFolderId) matched, err := regexp.MatchString(targetFolderRegexp, targetFolderIdString) @@ -372,6 +420,16 @@ func getSCCNotificationServiceOrTerminateExecution(ctx context.Context, client * return securitycenterClient } +func getAssetServiceOrTerminateExecution(ctx context.Context, client *http.Client) *asset.Client { + logger.Println("Try to get Asset Service") + assetService, err := asset.NewClient(ctx) + if err != nil { + logger.Fatalf("Failed to get Asset Service with error [%s], terminate execution", err.Error()) + } + logger.Println("Got Asset Service") + return assetService +} + func getFirewallPoliciesServiceOrTerminateExecution(ctx context.Context, client *http.Client) *compute.FirewallPoliciesService { logger.Println("Try to get Firewall Policies Service") computeService, err := compute.NewService(ctx, option.WithHTTPClient(client)) @@ -399,6 +457,7 @@ func invoke(ctx context.Context) { tagKeyService := getTagKeysServiceOrTerminateExecution(ctx, client) sccService := getSCCNotificationServiceOrTerminateExecution(ctx, client) tagValuesService := getTagValuesServiceOrTerminateExecution(ctx, client) + feedsService := getAssetServiceOrTerminateExecution(ctx, client) firewallPoliciesService := getFirewallPoliciesServiceOrTerminateExecution(ctx, client) endpointService := getServiceManagementServiceOrTerminateExecution(ctx, client) @@ -450,7 +509,7 @@ func invoke(ctx context.Context) { break } projectID := strings.Split(resp.PubsubTopic, "/")[1] - if checkIfSCCNotificationNameIncluded(resp.Name, includedSCCNotfisList) && projectDeleteRequestedFilter(projectID) { + if checkIfNameIncluded(resp.Name, includedSCCNotfisList) && projectDeleteRequestedFilter(projectID) { delReq := &securitycenterpb.DeleteNotificationConfigRequest{ Name: resp.Name, } @@ -498,6 +557,35 @@ func invoke(ctx context.Context) { } } + removeFeedsByName := func(organization string) { + logger.Printf("Try to remove feeds from organization [%s]", organization) + + req := &assetpb.ListFeedsRequest{ + Parent: fmt.Sprintf("organizations/%s", organization), + } + + resp, err := feedsService.ListFeeds(ctx, req) + if err != nil { + logger.Printf("Failed to list Feeds, error [%s]", err.Error()) + return + } + + for _, feed := range resp.Feeds { + projectID := strings.Split(feed.FeedOutputConfig.GetPubsubDestination().Topic, "/")[1] + if checkIfNameIncluded(feed.Name, includedFeedsList) && projectDeleteRequestedFilter(projectID) { + delReq := &assetpb.DeleteFeedRequest{ + Name: feed.Name, + } + err := feedsService.DeleteFeed(ctx, delReq) + if err != nil { + logger.Printf("Failed to remove the feed [%s], error [%s]", feed.Name, err.Error()) + } else { + logger.Printf("Feed [%s] successfully removed.", feed.Name) + } + } + } + } + removeFirewallPolicies := func(folder string) { logger.Printf("Try to remove Firewall Policies from folder [%s]", folder) firewallPolicyList, err := firewallPoliciesService.List().ParentId(folder).Context(ctx).Do() @@ -640,6 +728,7 @@ func invoke(ctx context.Context) { } else { getSubFoldersAndRemoveProjectsFoldersRecursively(rootFolder, getSubFoldersAndRemoveProjectsFoldersRecursively) } + // Only Tag Keys whose values are not in use can be deleted. if cleanUpTagKeys { removeTagKeys(organizationId) @@ -649,6 +738,11 @@ func invoke(ctx context.Context) { if cleanUpSCCNotfi { removeSCCNotifications(organizationId) } + + //Only delete Feeds from deleted projects + if cleanUpCaiFeeds { + removeFeedsByName(organizationId) + } } func CleanUpProjects(ctx context.Context, m PubSubMessage) error { diff --git a/modules/project_cleanup/main.tf b/modules/project_cleanup/main.tf index 654cdcb7..7195a0f3 100644 --- a/modules/project_cleanup/main.tf +++ b/modules/project_cleanup/main.tf @@ -34,6 +34,7 @@ resource "google_organization_iam_member" "main" { "roles/compute.orgSecurityPolicyAdmin", "roles/resourcemanager.tagAdmin", "roles/viewer", + "roles/cloudasset.owner", "roles/securitycenter.notificationConfigEditor" ]) @@ -69,5 +70,7 @@ module "scheduled_project_cleaner" { CLEAN_UP_SCC_NOTIFICATIONS = var.clean_up_org_level_scc_notifications TARGET_INCLUDED_SCC_NOTIFICATIONS = jsonencode(var.target_included_scc_notifications) SCC_NOTIFICATIONS_PAGE_SIZE = var.list_scc_notifications_page_size + CLEAN_UP_CAI_FEEDS = var.clean_up_org_level_cai_feeds + TARGET_INCLUDED_FEEDS = jsonencode(var.target_included_feeds) } } diff --git a/modules/project_cleanup/variables.tf b/modules/project_cleanup/variables.tf index 34720c92..91f51095 100644 --- a/modules/project_cleanup/variables.tf +++ b/modules/project_cleanup/variables.tf @@ -25,6 +25,18 @@ variable "organization_id" { description = "The organization ID whose projects to clean up" } +variable "clean_up_org_level_cai_feeds" { + type = bool + description = "Clean up organization level Cloud Asset Inventory Feeds." + default = false +} + +variable "target_included_feeds" { + type = list(string) + description = "List of organization level Cloud Asset Inventory feeds that should be deleted. Regex example: `.*/feeds/fd-cai-monitoring-.*` " + default = [] +} + variable "project_id" { type = string description = "The project ID to host the scheduled function in"