From 179cd9a7020417ebe790c929eedc42f2e91e1b5e Mon Sep 17 00:00:00 2001 From: Brent Barbachem Date: Wed, 13 Nov 2024 07:30:28 -0500 Subject: [PATCH] OCPBUGS-44193: Use client side filtering for gcp destroy ** The destroy code is using server side filtering on resources. The number of resources that are filtered out (server side) are causing quota limits to be reached. Moving the filtering to the client side will limit quota max errors. --- pkg/destroy/gcp/address.go | 92 ++++++++++--------- pkg/destroy/gcp/backendservice.go | 74 ++++++--------- pkg/destroy/gcp/bucket.go | 31 ++++--- pkg/destroy/gcp/bucketobject.go | 6 +- pkg/destroy/gcp/cloudcontroller.go | 130 ++++++++++++++++---------- pkg/destroy/gcp/disk.go | 37 ++++++-- pkg/destroy/gcp/filestore.go | 2 +- pkg/destroy/gcp/firewall.go | 25 ++--- pkg/destroy/gcp/forwardingrule.go | 137 +++++++++++++--------------- pkg/destroy/gcp/gcp.go | 16 +--- pkg/destroy/gcp/healthcheck.go | 133 +++++++++++++-------------- pkg/destroy/gcp/httphealthcheck.go | 20 ++-- pkg/destroy/gcp/image.go | 19 ++-- pkg/destroy/gcp/instance.go | 58 ++++++++---- pkg/destroy/gcp/instancegroup.go | 21 +++-- pkg/destroy/gcp/network.go | 25 ++--- pkg/destroy/gcp/route.go | 26 +++--- pkg/destroy/gcp/router.go | 19 ++-- pkg/destroy/gcp/subnetwork.go | 19 ++-- pkg/destroy/gcp/targetTCPProxies.go | 47 ++++++---- pkg/destroy/gcp/targetpool.go | 24 +++-- 21 files changed, 522 insertions(+), 439 deletions(-) diff --git a/pkg/destroy/gcp/address.go b/pkg/destroy/gcp/address.go index 19058d74241..19293430ce8 100644 --- a/pkg/destroy/gcp/address.go +++ b/pkg/destroy/gcp/address.go @@ -16,25 +16,53 @@ const ( ) func (o *ClusterUninstaller) listAddresses(ctx context.Context, typeName string) ([]cloudResource, error) { - return o.listAddressesWithFilter(ctx, typeName, "items(name,region,addressType),nextPageToken", o.clusterIDFilter()) + return o.listAddressesWithFilter(ctx, typeName, "items(name,region,addressType),nextPageToken", o.isClusterResource) } // listAddressesWithFilter lists addresses in the project that satisfy the filter criteria. // The fields parameter specifies which fields should be returned in the result, the filter string contains // a filter string passed to the API to filter results. -func (o *ClusterUninstaller) listAddressesWithFilter(ctx context.Context, typeName, fields, filter string) ([]cloudResource, error) { +func (o *ClusterUninstaller) listAddressesWithFilter(ctx context.Context, typeName, fields string, filterFunc resourceFilterFunc) ([]cloudResource, error) { o.Logger.Debugf("Listing addresses") + result := []cloudResource{} + + pagesFunc := func(list *compute.AddressList) error { + for _, item := range list.Items { + o.Logger.Debugf("Found address (%s): %s", typeName, item.Name) + if filterFunc(item.Name) { + var quota []gcp.QuotaUsage + if item.AddressType == "INTERNAL" { + quota = []gcp.QuotaUsage{{ + Metric: &gcp.Metric{ + Service: gcp.ServiceComputeEngineAPI, + Limit: "internal_addresses", + Dimensions: map[string]string{ + "region": getNameFromURL("regions", item.Region), + }, + }, + Amount: 1, + }} + } + result = append(result, cloudResource{ + key: item.Name, + name: item.Name, + typeName: typeName, + quota: quota, + }) + } + } + return nil + } ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() var err error - var list *compute.AddressList switch typeName { case globalAddressResource: - list, err = o.computeSvc.GlobalAddresses.List(o.ProjectID).Filter(filter).Fields(googleapi.Field(fields)).Context(ctx).Do() + err = o.computeSvc.GlobalAddresses.List(o.ProjectID).Fields(googleapi.Field(fields)).Pages(ctx, pagesFunc) case regionalAddressResource: - list, err = o.computeSvc.Addresses.List(o.ProjectID, o.Region).Filter(filter).Fields(googleapi.Field(fields)).Context(ctx).Do() + err = o.computeSvc.Addresses.List(o.ProjectID, o.Region).Fields(googleapi.Field(fields)).Pages(ctx, pagesFunc) default: return nil, fmt.Errorf("invalid address type %q", typeName) } @@ -43,25 +71,6 @@ func (o *ClusterUninstaller) listAddressesWithFilter(ctx context.Context, typeNa return nil, fmt.Errorf("failed to list addresses: %w", err) } - result := []cloudResource{} - for _, item := range list.Items { - o.Logger.Debugf("Found address: %s, type: %s", item.Name, item.AddressType) - result = append(result, cloudResource{ - key: item.Name, - name: item.Name, - typeName: typeName, - quota: []gcp.QuotaUsage{{ - Metric: &gcp.Metric{ - Service: gcp.ServiceComputeEngineAPI, - Limit: "addresses", - Dimensions: map[string]string{ - "region": getNameFromURL("regions", item.Region), - }, - }, - Amount: 1, - }}, - }) - } return result, nil } @@ -100,31 +109,24 @@ func (o *ClusterUninstaller) deleteAddress(ctx context.Context, item cloudResour // destroyAddresses removes all address resources that have a name prefixed // with the cluster's infra ID. func (o *ClusterUninstaller) destroyAddresses(ctx context.Context) error { - found, err := o.listAddresses(ctx, globalAddressResource) - if err != nil { - return err - } - items := o.insertPendingItems(globalAddressResource, found) - - found, err = o.listAddresses(ctx, regionalAddressResource) - if err != nil { - return err - } - items = append(items, o.insertPendingItems(regionalAddressResource, found)...) - - for _, item := range items { - err := o.deleteAddress(ctx, item) + addressTypes := []string{globalAddressResource, regionalAddressResource} + for _, addressType := range addressTypes { + found, err := o.listAddresses(ctx, addressType) if err != nil { - o.errorTracker.suppressWarning(item.key, err, o.Logger) + return err } - } + items := o.insertPendingItems(addressType, found) - if items = o.getPendingItems(globalAddressResource); len(items) > 0 { - return fmt.Errorf("%d global addresses pending", len(items)) - } + for _, item := range items { + err := o.deleteAddress(ctx, item) + if err != nil { + o.errorTracker.suppressWarning(item.key, err, o.Logger) + } + } - if items = o.getPendingItems(regionalAddressResource); len(items) > 0 { - return fmt.Errorf("%d region addresses pending", len(items)) + if items := o.getPendingItems(addressType); len(items) > 0 { + return fmt.Errorf("%d %s resources pending", len(items), addressType) + } } return nil diff --git a/pkg/destroy/gcp/backendservice.go b/pkg/destroy/gcp/backendservice.go index 07945f24bf8..82c1c46bfbf 100644 --- a/pkg/destroy/gcp/backendservice.go +++ b/pkg/destroy/gcp/backendservice.go @@ -6,7 +6,6 @@ import ( "google.golang.org/api/compute/v1" "google.golang.org/api/googleapi" - "k8s.io/apimachinery/pkg/util/sets" "github.com/openshift/installer/pkg/types/gcp" ) @@ -17,41 +16,48 @@ const ( ) func (o *ClusterUninstaller) listBackendServices(ctx context.Context, typeName string) ([]cloudResource, error) { - return o.listBackendServicesWithFilter(ctx, typeName, "items(name),nextPageToken", o.clusterIDFilter(), nil) -} - -func backendServiceBelongsToInstanceGroup(item *compute.BackendService, igURLs sets.Set[string]) bool { - if igURLs == nil { - return true - } - - if len(item.Backends) == 0 { - return false - } - for _, backend := range item.Backends { - if !igURLs.Has(backend.Group) { - return false - } - } - return true + return o.listBackendServicesWithFilter(ctx, typeName, "items(name),nextPageToken", + func(item *compute.BackendService) bool { + return o.isClusterResource(item.Name) + }) } // listBackendServicesWithFilter lists backend services in the project that satisfy the filter criteria. -// The fields parameter specifies which fields should be returned in the result, the filter string contains -// a filter string passed to the API to filter results. -func (o *ClusterUninstaller) listBackendServicesWithFilter(ctx context.Context, typeName, fields, filter string, urls sets.Set[string]) ([]cloudResource, error) { +// The fields parameter specifies which fields should be returned in the result. +func (o *ClusterUninstaller) listBackendServicesWithFilter(ctx context.Context, typeName, fields string, filterFunc func(item *compute.BackendService) bool) ([]cloudResource, error) { o.Logger.Debugf("Listing backend services") + result := []cloudResource{} ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() + pagesFunc := func(list *compute.BackendServiceList) error { + for _, item := range list.Items { + o.Logger.Debugf("Found backend service: %s", item.Name) + if filterFunc(item) { + result = append(result, cloudResource{ + key: item.Name, + name: item.Name, + typeName: typeName, + quota: []gcp.QuotaUsage{{ + Metric: &gcp.Metric{ + Service: gcp.ServiceComputeEngineAPI, + Limit: "backend_services", + }, + Amount: 1, + }}, + }) + } + } + return nil + } + var err error - var list *compute.BackendServiceList switch typeName { case globalBackendServiceResource: - list, err = o.computeSvc.BackendServices.List(o.ProjectID).Filter(filter).Fields(googleapi.Field(fields)).Context(ctx).Do() + err = o.computeSvc.BackendServices.List(o.ProjectID).Fields(googleapi.Field(fields)).Pages(ctx, pagesFunc) case regionBackendServiceResource: - list, err = o.computeSvc.RegionBackendServices.List(o.ProjectID, o.Region).Filter(filter).Fields(googleapi.Field(fields)).Context(ctx).Do() + err = o.computeSvc.RegionBackendServices.List(o.ProjectID, o.Region).Fields(googleapi.Field(fields)).Pages(ctx, pagesFunc) default: return nil, fmt.Errorf("invalid backend service type %q", typeName) } @@ -60,26 +66,6 @@ func (o *ClusterUninstaller) listBackendServicesWithFilter(ctx context.Context, return nil, fmt.Errorf("failed to list backend services: %w", err) } - result := []cloudResource{} - for _, item := range list.Items { - o.Logger.Debugf("Found backend service: %s", item.Name) - if !backendServiceBelongsToInstanceGroup(item, urls) { - o.Logger.Debug("No matching instance group for backend service: %s", item.Name) - continue - } - result = append(result, cloudResource{ - key: item.Name, - name: item.Name, - typeName: typeName, - quota: []gcp.QuotaUsage{{ - Metric: &gcp.Metric{ - Service: gcp.ServiceComputeEngineAPI, - Limit: "backend_services", - }, - Amount: 1, - }}, - }) - } return result, nil } diff --git a/pkg/destroy/gcp/bucket.go b/pkg/destroy/gcp/bucket.go index 9df2e53ee86..a5e5dc63f3d 100644 --- a/pkg/destroy/gcp/bucket.go +++ b/pkg/destroy/gcp/bucket.go @@ -2,7 +2,9 @@ package gcp import ( "context" + "fmt" "regexp" + "strings" "github.com/pkg/errors" "google.golang.org/api/googleapi" @@ -14,39 +16,44 @@ var ( multiDashes = regexp.MustCompile(`-{2,}`) ) +const ( + bucketResourceName = "bucket" +) + func (o *ClusterUninstaller) listBuckets(ctx context.Context) ([]cloudResource, error) { - return o.listBucketsWithFilter(ctx, "items(name),nextPageToken", o.ClusterID+"-", nil) + return o.listBucketsWithFilter(ctx, "items(name),nextPageToken", + func(itemName string) bool { + prefix := multiDashes.ReplaceAllString(o.ClusterID+"-", "-") + return strings.HasPrefix(itemName, prefix) + }) } // listBucketsWithFilter lists buckets in the project that satisfy the filter criteria. // The fields parameter specifies which fields should be returned in the result, the filter string contains // a prefix string passed to the API to filter results. The filterFunc is a client-side filtering function // that determines whether a particular result should be returned or not. -func (o *ClusterUninstaller) listBucketsWithFilter(ctx context.Context, fields string, prefix string, filterFunc func(*storage.Bucket) bool) ([]cloudResource, error) { +func (o *ClusterUninstaller) listBucketsWithFilter(ctx context.Context, fields string, filterFunc resourceFilterFunc) ([]cloudResource, error) { o.Logger.Debug("Listing storage buckets") ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() result := []cloudResource{} req := o.storageSvc.Buckets.List(o.ProjectID).Fields(googleapi.Field(fields)) - if len(prefix) > 0 { - prefix = multiDashes.ReplaceAllString(prefix, "-") - req = req.Prefix(prefix) - } + err := req.Pages(ctx, func(list *storage.Buckets) error { for _, item := range list.Items { - if filterFunc == nil || filterFunc != nil && filterFunc(item) { + if filterFunc(item.Name) { o.Logger.Debugf("Found bucket: %s", item.Name) result = append(result, cloudResource{ key: item.Name, name: item.Name, - typeName: "bucket", + typeName: bucketResourceName, }) } } return nil }) if err != nil { - return nil, errors.Wrap(err, "failed to fetch object storage buckets") + return nil, fmt.Errorf("failed to fetch object storage buckets: %w", err) } return result, nil } @@ -71,13 +78,13 @@ func (o *ClusterUninstaller) destroyBuckets(ctx context.Context) error { if err != nil { return err } - items := o.insertPendingItems("bucket", found) + items := o.insertPendingItems(bucketResourceName, found) for _, item := range items { foundObjects, err := o.listBucketObjects(ctx, item) if err != nil { return err } - objects := o.insertPendingItems("bucketobject", foundObjects) + objects := o.insertPendingItems(bucketObjectResourceName, foundObjects) for _, object := range objects { err = o.deleteBucketObject(ctx, item, object) if err != nil { @@ -89,7 +96,7 @@ func (o *ClusterUninstaller) destroyBuckets(ctx context.Context) error { o.errorTracker.suppressWarning(item.key, err, o.Logger) } } - if items = o.getPendingItems("bucket"); len(items) > 0 { + if items = o.getPendingItems(bucketResourceName); len(items) > 0 { return errors.Errorf("%d items pending", len(items)) } return nil diff --git a/pkg/destroy/gcp/bucketobject.go b/pkg/destroy/gcp/bucketobject.go index 04bedb037b0..bb9d922a7f3 100644 --- a/pkg/destroy/gcp/bucketobject.go +++ b/pkg/destroy/gcp/bucketobject.go @@ -7,6 +7,10 @@ import ( storage "google.golang.org/api/storage/v1" ) +const ( + bucketObjectResourceName = "bucketobject" +) + func (o *ClusterUninstaller) listBucketObjects(ctx context.Context, bucket cloudResource) ([]cloudResource, error) { o.Logger.Debugf("Listing objects for storage bucket %s", bucket.name) ctx, cancel := context.WithTimeout(ctx, defaultTimeout) @@ -19,7 +23,7 @@ func (o *ClusterUninstaller) listBucketObjects(ctx context.Context, bucket cloud result = append(result, cloudResource{ key: object.Name, name: object.Name, - typeName: "bucketobject", + typeName: bucketObjectResourceName, }) } return nil diff --git a/pkg/destroy/gcp/cloudcontroller.go b/pkg/destroy/gcp/cloudcontroller.go index c1f0496723d..ebff03efdc0 100644 --- a/pkg/destroy/gcp/cloudcontroller.go +++ b/pkg/destroy/gcp/cloudcontroller.go @@ -3,42 +3,70 @@ package gcp import ( "context" "fmt" + "regexp" + "strings" - compute "google.golang.org/api/compute/v1" + "google.golang.org/api/compute/v1" "k8s.io/apimachinery/pkg/util/sets" ) +type resourceFilterFunc func(string) bool + +func (o *ClusterUninstaller) createLoadBalancerFilterFunc(loadBalancerName string) resourceFilterFunc { + return func(itemName string) bool { + return strings.HasPrefix(itemName, loadBalancerName) + } +} + // listCloudControllerInstanceGroups returns instance groups created by the cloud controller. // It list all instance groups matching the cloud controller name convention. // https://github.com/openshift/kubernetes/blob/1e5983903742f64bca36a464582178c940353e9a/pkg/cloudprovider/providers/gce/gce_loadbalancer_naming.go#L33-L40 // https://github.com/openshift/kubernetes/blob/1e5983903742f64bca36a464582178c940353e9a/pkg/cloudprovider/providers/gce/gce_clusterid.go#L210-L238 func (o *ClusterUninstaller) listCloudControllerInstanceGroups(ctx context.Context) ([]cloudResource, error) { - filter := fmt.Sprintf("name eq \"k8s-ig--%s\"", o.cloudControllerUID) - return o.listInstanceGroupsWithFilter(ctx, "items/*/instanceGroups(name,selfLink,zone),nextPageToken", filter, nil) + return o.listInstanceGroupsWithFilter(ctx, "items/*/instanceGroups(name,selfLink,zone),nextPageToken", func(itemName string) bool { + // TODO: Why does this have an extra - ?? + return itemName == fmt.Sprintf("k8s-ig--%s", o.cloudControllerUID) + }) } // listCloudControllerBackendServices returns backend services created by the cloud controller. // It list all backend services matching the cloud controller name convention that contain // only cluster instance groups. func (o *ClusterUninstaller) listCloudControllerBackendServices(ctx context.Context, instanceGroups []cloudResource) ([]cloudResource, error) { - urls := sets.Set[string]{} - for _, instanceGroup := range instanceGroups { - urls.Insert(instanceGroup.url) - } - filter := "name eq \"a[0-9a-f]{30,50}\"" - return o.listBackendServicesWithFilter(ctx, regionBackendServiceResource, "items(name,backends),nextPageToken", filter, urls) + return o.listBackendServicesWithFilter(ctx, regionBackendServiceResource, "items(name,backends),nextPageToken", + func(item *compute.BackendService) bool { + filter := regexp.MustCompile(`a[0-9a-f]{30,50}`) + if !filter.MatchString(item.Name) { + return false + } + + urls := sets.Set[string]{} + for _, instanceGroup := range instanceGroups { + urls.Insert(instanceGroup.url) + } + + if len(item.Backends) == 0 { + return false + } + for _, backend := range item.Backends { + if !urls.Has(backend.Group) { + return false + } + } + return true + }) } // listCloudControllerTargetPools returns target pools created by the cloud controller or owned by the cloud controller. // It lists all target pools matching the cloud controller name convention that contain // only cluster instances or cluster instances that were owned by the cluster. func (o *ClusterUninstaller) listCloudControllerTargetPools(ctx context.Context, instances []cloudResource) ([]cloudResource, error) { - filter := "name eq \"a[0-9a-f]{30,50}\"" - return o.listTargetPoolsWithFilter(ctx, "items(name,instances),nextPageToken", filter, func(pool *compute.TargetPool) bool { - if len(pool.Instances) == 0 { + return o.listTargetPoolsWithFilter(ctx, "items(name,instances),nextPageToken", func(item *compute.TargetPool) bool { + filter := regexp.MustCompile(`a[0-9a-f]{30,50}`) + if !filter.MatchString(item.Name) || len(item.Instances) == 0 { return false } - for _, instanceURL := range pool.Instances { + for _, instanceURL := range item.Instances { name, _ := o.getInstanceNameAndZone(instanceURL) if !o.isClusterResource(name) { foundClusterResource := false @@ -50,7 +78,7 @@ func (o *ClusterUninstaller) listCloudControllerTargetPools(ctx context.Context, } if !foundClusterResource { - o.Logger.Debugf("Skipping target pool instance %s because it is not a cluster resource", pool.Name) + o.Logger.Debugf("Skipping target pool instance %s because it is not a cluster resource", item.Name) return false } } @@ -63,73 +91,74 @@ func (o *ClusterUninstaller) listCloudControllerTargetPools(ctx context.Context, // https://github.com/openshift/kubernetes/blob/1e5983903742f64bca36a464582178c940353e9a/pkg/cloudprovider/providers/gce/gce_loadbalancer_internal.go#L222 // https://github.com/openshift/kubernetes/blob/1e5983903742f64bca36a464582178c940353e9a/pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go#L289 func (o *ClusterUninstaller) discoverCloudControllerLoadBalancerResources(ctx context.Context, loadBalancerName string) error { - loadBalancerNameFilter := fmt.Sprintf("name eq \"%s\"", loadBalancerName) + loadBalancerFilterFunc := o.createLoadBalancerFilterFunc(loadBalancerName) // Discover associated addresses: loadBalancerName - found, err := o.listAddressesWithFilter(ctx, "regionaddress", "items(name),nextPageToken", loadBalancerNameFilter) + found, err := o.listAddressesWithFilter(ctx, regionalAddressResource, "items(name),nextPageToken", loadBalancerFilterFunc) if err != nil { return err } - o.insertPendingItems("address", found) + o.insertPendingItems(regionalAddressResource, found) // Discover associated firewall rules: loadBalancerName - found, err = o.listFirewallsWithFilter(ctx, "items(name),nextPageToken", loadBalancerNameFilter, nil) + found, err = o.listFirewallsWithFilter(ctx, "items(name),nextPageToken", loadBalancerFilterFunc) if err != nil { return err } - o.insertPendingItems("firewall", found) + o.insertPendingItems(firewallResourceName, found) // Discover associated firewall rules: loadBalancerName-hc - filter := fmt.Sprintf("name eq \"%s-hc\"", loadBalancerName) - found, err = o.listFirewallsWithFilter(ctx, "items(name),nextPageToken", filter, nil) + found, err = o.listFirewallsWithFilter(ctx, "items(name),nextPageToken", o.createLoadBalancerFilterFunc(fmt.Sprintf("%s-hc", loadBalancerName))) if err != nil { return err } - o.insertPendingItems("firewall", found) + o.insertPendingItems(firewallResourceName, found) // Discover associated firewall rules: k8s-fw-loadBalancerName - filter = fmt.Sprintf("name eq \"k8s-fw-%s\"", loadBalancerName) - found, err = o.listFirewallsWithFilter(ctx, "items(name),nextPageToken", filter, nil) + found, err = o.listFirewallsWithFilter(ctx, "items(name),nextPageToken", + o.createLoadBalancerFilterFunc(fmt.Sprintf("k8s-fw-%s", loadBalancerName)), + ) if err != nil { return err } - o.insertPendingItems("firewall", found) + o.insertPendingItems(firewallResourceName, found) // Discover associated firewall rules: k8s-loadBalancerName-http-hc - filter = fmt.Sprintf("name eq \"k8s-%s-http-hc\"", loadBalancerName) - found, err = o.listFirewallsWithFilter(ctx, "items(name),nextPageToken", filter, nil) + found, err = o.listFirewallsWithFilter(ctx, "items(name),nextPageToken", + o.createLoadBalancerFilterFunc(fmt.Sprintf("k8s-%s-http-hc", loadBalancerName)), + ) if err != nil { return err } - o.insertPendingItems("firewall", found) + o.insertPendingItems(firewallResourceName, found) // Discover associated forwarding rules: loadBalancerName - found, err = o.listForwardingRulesWithFilter(ctx, "items(name),nextPageToken", loadBalancerNameFilter, nil, gcpRegionalResource) + found, err = o.listForwardingRulesWithFilter(ctx, regionForwardingRuleResource, "items(name),nextPageToken", loadBalancerFilterFunc) if err != nil { return err } - o.insertPendingItems("forwardingrule", found) + o.insertPendingItems(regionForwardingRuleResource, found) // Discover associated target tcp proxies: loadBalancerName - found, err = o.listTargetTCPProxiesWithFilter(ctx, globalTargetTCPProxyResource, "items(name),nextPageToken", loadBalancerNameFilter) + found, err = o.listTargetTCPProxiesWithFilter(ctx, globalTargetTCPProxyResource, "items(name),nextPageToken", loadBalancerFilterFunc) if err != nil { return err } o.insertPendingItems(globalTargetTCPProxyResource, found) // Discover associated health checks: loadBalancerName - found, err = o.listHealthChecksWithFilter(ctx, "healthcheck", "items(name),nextPageToken", loadBalancerNameFilter, o.healthCheckList) + found, err = o.listHealthChecksWithFilter(ctx, regionHealthCheckResource, "items(name),nextPageToken", loadBalancerFilterFunc) if err != nil { return err } - o.insertPendingItems("healthcheck", found) + o.insertPendingItems(regionHealthCheckResource, found) // Discover associated http health checks: loadBalancerName - found, err = o.listHTTPHealthChecksWithFilter(ctx, "items(name),nextPageToken", loadBalancerNameFilter, nil) + found, err = o.listHTTPHealthChecksWithFilter(ctx, "items(name),nextPageToken", loadBalancerFilterFunc) if err != nil { return err } - o.insertPendingItems("httphealthcheck", found) + o.insertPendingItems(httpHealthCheckResourceName, found) return nil } @@ -167,7 +196,7 @@ func (o *ClusterUninstaller) discoverCloudControllerResources(ctx context.Contex } o.insertPendingItems("backendservice", backends) } - o.insertPendingItems("instancegroup", instanceGroups) + o.insertPendingItems(instanceGroupResourceName, instanceGroups) // Get a list of known cluster instances instances, err := o.listInstances(ctx) @@ -187,39 +216,44 @@ func (o *ClusterUninstaller) discoverCloudControllerResources(ctx context.Contex errs = append(errs, err) } } - o.insertPendingItems("targetpool", pools) + o.insertPendingItems(targetPoolResourceName, pools) // cloudControllerUID related items if len(o.cloudControllerUID) > 0 { // Discover Cloud Controller health checks: k8s-cloudControllerUID-node - filter := fmt.Sprintf("name eq \"k8s-%s-node\"", o.cloudControllerUID) - found, err := o.listHealthChecksWithFilter(ctx, "healthcheck", "items(name),nextPageToken", filter, o.healthCheckList) + found, err := o.listHealthChecksWithFilter(ctx, regionHealthCheckResource, "items(name),nextPageToken", + o.createLoadBalancerFilterFunc(fmt.Sprintf("k8s-%s-node", o.cloudControllerUID)), + ) if err != nil { return err } - o.insertPendingItems("healthcheck", found) + o.insertPendingItems(regionHealthCheckResource, found) // Discover Cloud Controller http health checks: k8s-cloudControllerUID-node - found, err = o.listHTTPHealthChecksWithFilter(ctx, "items(name),nextPageToken", filter, nil) + found, err = o.listHTTPHealthChecksWithFilter(ctx, "items(name),nextPageToken", + o.createLoadBalancerFilterFunc(fmt.Sprintf("k8s-%s-node", o.cloudControllerUID)), + ) if err != nil { return err } - o.insertPendingItems("httphealthcheck", found) + o.insertPendingItems(httpHealthCheckResourceName, found) // Discover Cloud Controller firewall rules: k8s-cloudControllerUID-node-hc, k8s-cloudControllerUID-node-http-hc - filter = fmt.Sprintf("name eq \"k8s-%s-node-hc\"", o.cloudControllerUID) - found, err = o.listFirewallsWithFilter(ctx, "items(name),nextPageToken", filter, nil) + found, err = o.listFirewallsWithFilter(ctx, "items(name),nextPageToken", + o.createLoadBalancerFilterFunc(fmt.Sprintf("k8s-%s-node-hc", o.cloudControllerUID)), + ) if err != nil { return err } - o.insertPendingItems("firewall", found) + o.insertPendingItems(firewallResourceName, found) - filter = fmt.Sprintf("name eq \"k8s-%s-node-http-hc\"", o.cloudControllerUID) - found, err = o.listFirewallsWithFilter(ctx, "items(name),nextPageToken", filter, nil) + found, err = o.listFirewallsWithFilter(ctx, "items(name),nextPageToken", + o.createLoadBalancerFilterFunc(fmt.Sprintf("k8s-%s-node-http-hc", o.cloudControllerUID)), + ) if err != nil { return err } - o.insertPendingItems("firewall", found) + o.insertPendingItems(firewallResourceName, found) } return aggregateError(errs, 0) diff --git a/pkg/destroy/gcp/disk.go b/pkg/destroy/gcp/disk.go index d68e3dc1d8b..6c1aff4943d 100644 --- a/pkg/destroy/gcp/disk.go +++ b/pkg/destroy/gcp/disk.go @@ -3,6 +3,7 @@ package gcp import ( "context" "fmt" + "strings" "github.com/pkg/errors" "google.golang.org/api/compute/v1" @@ -19,6 +20,10 @@ const ( storageNameLength = maxGCEPDNameLength - estimatedPVNameLength - 1 ) +const ( + diskResourceName = "disk" +) + // formatClusterIDForStorage will format the Cluster ID as it will be used for destroying // GCE PDs. The maximum length is 63 characters, and can end with "-dynamic". // https://github.com/kubernetes/kubernetes/blob/master/pkg/volume/util/util.go, GenerateVolumeName() @@ -36,7 +41,7 @@ func (o *ClusterUninstaller) storageIDFilter() string { } func (o *ClusterUninstaller) storageLabelFilter() string { - return fmt.Sprintf("labels.%s = \"owned\"", fmt.Sprintf(gcpconsts.ClusterIDLabelFmt, o.formatClusterIDForStorage())) + return fmt.Sprintf("labels.%s = \"%s\"", ownedLabelValue, fmt.Sprintf(gcpconsts.ClusterIDLabelFmt, o.formatClusterIDForStorage())) } // storageLabelOrClusterIDFilter will perform the search for resources with the ClusterID, but @@ -46,26 +51,38 @@ func (o *ClusterUninstaller) storageLabelOrClusterIDFilter() string { } func (o *ClusterUninstaller) listDisks(ctx context.Context) ([]cloudResource, error) { - return o.listDisksWithFilter(ctx, "items/*/disks(name,zone,type,sizeGb),nextPageToken", o.storageLabelOrClusterIDFilter(), nil) + return o.listDisksWithFilter(ctx, "items/*/disks(name,zone,type,sizeGb),nextPageToken", func(item *compute.Disk) bool { + if o.isClusterResource(item.Name) || strings.HasPrefix(item.Name, o.formatClusterIDForStorage()) { + return true + } + + // TODO: do labels get formatted as labels.%s, or is this only for filters + for key, value := range item.Labels { + if key == fmt.Sprintf(gcpconsts.ClusterIDLabelFmt, o.ClusterID) && value == ownedLabelValue { + return true + } else if key == fmt.Sprintf(capgProviderOwnedLabelFmt, o.ClusterID) && value == ownedLabelValue { + return true + } + } + return false + }) } // listDisksWithFilter lists disks in the project that satisfy the filter criteria. // The fields parameter specifies which fields should be returned in the result, the filter string contains // a filter string passed to the API to filter results. The filterFunc is a client-side filtering function // that determines whether a particular result should be returned or not. -func (o *ClusterUninstaller) listDisksWithFilter(ctx context.Context, fields string, filter string, filterFunc func(*compute.Disk) bool) ([]cloudResource, error) { +func (o *ClusterUninstaller) listDisksWithFilter(ctx context.Context, fields string, filterFunc func(*compute.Disk) bool) ([]cloudResource, error) { o.Logger.Debug("Listing disks") ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() result := []cloudResource{} req := o.computeSvc.Disks.AggregatedList(o.ProjectID).Fields(googleapi.Field(fields)) - if len(filter) > 0 { - req = req.Filter(filter) - } + err := req.Pages(ctx, func(list *compute.DiskAggregatedList) error { for _, scopedList := range list.Items { for _, item := range scopedList.Disks { - if filterFunc == nil || filterFunc != nil && filterFunc(item) { + if filterFunc(item) { // Regional disks are replicated in multiple zones, so we // need to destroy all the replicas zoneUrls := item.ReplicaZones @@ -78,7 +95,7 @@ func (o *ClusterUninstaller) listDisksWithFilter(ctx context.Context, fields str result = append(result, cloudResource{ key: fmt.Sprintf("%s/%s", zone, item.Name), name: item.Name, - typeName: "disk", + typeName: diskResourceName, zone: zone, quota: []gcp.QuotaUsage{{ Metric: &gcp.Metric{ @@ -131,14 +148,14 @@ func (o *ClusterUninstaller) destroyDisks(ctx context.Context) error { if err != nil { return err } - items := o.insertPendingItems("disk", found) + items := o.insertPendingItems(diskResourceName, found) for _, item := range items { err := o.deleteDisk(ctx, item) if err != nil { o.errorTracker.suppressWarning(item.key, err, o.Logger) } } - if items = o.getPendingItems("disk"); len(items) > 0 { + if items = o.getPendingItems(diskResourceName); len(items) > 0 { return errors.Errorf("%d items pending", len(items)) } return nil diff --git a/pkg/destroy/gcp/filestore.go b/pkg/destroy/gcp/filestore.go index 374cdacd1f5..00fcb081f19 100644 --- a/pkg/destroy/gcp/filestore.go +++ b/pkg/destroy/gcp/filestore.go @@ -14,7 +14,7 @@ func (o *ClusterUninstaller) filestoreParentPath() string { } func (o *ClusterUninstaller) clusterFilestoreLabelFilter() string { - return fmt.Sprintf("labels.%s = \"owned\"", fmt.Sprintf(gcpconsts.ClusterIDLabelFmt, o.ClusterID)) + return fmt.Sprintf("labels.%s = \"%s\"", fmt.Sprintf(gcpconsts.ClusterIDLabelFmt, o.ClusterID), ownedLabelValue) } func (o *ClusterUninstaller) listFilestores(ctx context.Context) ([]cloudResource, error) { diff --git a/pkg/destroy/gcp/firewall.go b/pkg/destroy/gcp/firewall.go index 4467b345f26..83ea15757d9 100644 --- a/pkg/destroy/gcp/firewall.go +++ b/pkg/destroy/gcp/firewall.go @@ -2,7 +2,7 @@ package gcp import ( "context" - "fmt" + "strings" "github.com/pkg/errors" "google.golang.org/api/compute/v1" @@ -11,19 +11,24 @@ import ( "github.com/openshift/installer/pkg/types/gcp" ) +const ( + firewallResourceName = "firewall" +) + func (o *ClusterUninstaller) listFirewalls(ctx context.Context) ([]cloudResource, error) { // The firewall rules that the destroyer is searching for here include a // pattern before and after the cluster ID. Use a regular expression that allows // wildcard values before and after the cluster ID. - filter := fmt.Sprintf("name eq .*%s.*", o.ClusterID) - return o.listFirewallsWithFilter(ctx, "items(name),nextPageToken", filter, nil) + return o.listFirewallsWithFilter(ctx, "items(name),nextPageToken", func(item string) bool { + return strings.Contains(item, o.ClusterID) + }) } // listFirewallsWithFilter lists firewall rules in the project that satisfy the filter criteria. // The fields parameter specifies which fields should be returned in the result, the filter string contains // a filter string passed to the API to filter results. The filterFunc is a client-side filtering function // that determines whether a particular result should be returned or not. -func (o *ClusterUninstaller) listFirewallsWithFilter(ctx context.Context, fields string, filter string, filterFunc func(*compute.Firewall) bool) ([]cloudResource, error) { +func (o *ClusterUninstaller) listFirewallsWithFilter(ctx context.Context, fields string, filterFunc resourceFilterFunc) ([]cloudResource, error) { o.Logger.Debugf("Listing firewall rules") results := []cloudResource{} @@ -32,18 +37,16 @@ func (o *ClusterUninstaller) listFirewallsWithFilter(ctx context.Context, fields defer cancel() result := []cloudResource{} req := o.computeSvc.Firewalls.List(projectID).Fields(googleapi.Field(fields)) - if len(filter) > 0 { - req = req.Filter(filter) - } + err := req.Pages(ctx, func(list *compute.FirewallList) error { for _, item := range list.Items { - if filterFunc == nil || filterFunc != nil && filterFunc(item) { + if filterFunc(item.Name) { o.Logger.Debugf("Found firewall rule: %s", item.Name) result = append(result, cloudResource{ key: item.Name, name: item.Name, project: projectID, - typeName: "firewall", + typeName: firewallResourceName, quota: []gcp.QuotaUsage{{ Metric: &gcp.Metric{ Service: gcp.ServiceComputeEngineAPI, @@ -109,14 +112,14 @@ func (o *ClusterUninstaller) destroyFirewalls(ctx context.Context) error { if err != nil { return err } - items := o.insertPendingItems("firewall", found) + items := o.insertPendingItems(firewallResourceName, found) for _, item := range items { err := o.deleteFirewall(ctx, item) if err != nil { o.errorTracker.suppressWarning(item.key, err, o.Logger) } } - if items = o.getPendingItems("firewall"); len(items) > 0 { + if items = o.getPendingItems(firewallResourceName); len(items) > 0 { return errors.Errorf("%d items pending", len(items)) } return nil diff --git a/pkg/destroy/gcp/forwardingrule.go b/pkg/destroy/gcp/forwardingrule.go index 3913749a134..6c4e6c042c2 100644 --- a/pkg/destroy/gcp/forwardingrule.go +++ b/pkg/destroy/gcp/forwardingrule.go @@ -11,95 +11,85 @@ import ( "github.com/openshift/installer/pkg/types/gcp" ) -func (o *ClusterUninstaller) listForwardingRules(ctx context.Context, scope resourceScope) ([]cloudResource, error) { - return o.listForwardingRulesWithFilter(ctx, "items(name,region,loadBalancingScheme),nextPageToken", o.clusterIDFilter(), nil, scope) -} - -func createForwardingRuleResources(filterFunc func(*compute.ForwardingRule) bool, list *compute.ForwardingRuleList) []cloudResource { - result := []cloudResource{} - - for _, item := range list.Items { - if filterFunc == nil || filterFunc(item) { - logrus.Debugf("Found forwarding rule: %s", item.Name) - var quota []gcp.QuotaUsage - if item.LoadBalancingScheme == "EXTERNAL" { - quota = []gcp.QuotaUsage{{ - Metric: &gcp.Metric{ - Service: gcp.ServiceComputeEngineAPI, - Limit: "external_network_lb_forwarding_rules", - Dimensions: map[string]string{ - "region": getNameFromURL("regions", item.Region), - }, - }, - Amount: 1, - }} - } - result = append(result, cloudResource{ - key: item.Name, - name: item.Name, - typeName: "forwardingrule", - quota: quota, - }) - } - } +const ( + globalForwardingRuleResource = "fowardingrule" + regionForwardingRuleResource = "regionalforwardingrule" +) - return result +func (o *ClusterUninstaller) listForwardingRules(ctx context.Context, typeName string) ([]cloudResource, error) { + return o.listForwardingRulesWithFilter(ctx, typeName, "items(name,region,loadBalancingScheme),nextPageToken", o.isClusterResource) } // listForwardingRulesWithFilter lists forwarding rules in the project that satisfy the filter criteria. // The fields parameter specifies which fields should be returned in the result, the filter string contains // a filter string passed to the API to filter results. The filterFunc is a client-side filtering function // that determines whether a particular result should be returned or not. -func (o *ClusterUninstaller) listForwardingRulesWithFilter(ctx context.Context, fields string, filter string, filterFunc func(*compute.ForwardingRule) bool, scope resourceScope) ([]cloudResource, error) { - o.Logger.Debugf("Listing %s forwarding rules", scope) +func (o *ClusterUninstaller) listForwardingRulesWithFilter(ctx context.Context, typeName, fields string, filterFunc resourceFilterFunc) ([]cloudResource, error) { + o.Logger.Debugf("Listing forwarding rules") ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() result := []cloudResource{} - if scope == gcpGlobalResource { - req := o.computeSvc.GlobalForwardingRules.List(o.ProjectID).Fields(googleapi.Field(fields)) - if len(filter) > 0 { - req = req.Filter(filter) - } - err := req.Pages(ctx, func(list *compute.ForwardingRuleList) error { - result = append(result, createForwardingRuleResources(filterFunc, list)...) - return nil - }) - if err != nil { - return nil, fmt.Errorf("failed to list global forwarding rules: %w", err) + pagesFunc := func(list *compute.ForwardingRuleList) error { + for _, item := range list.Items { + if filterFunc(item.Name) { + logrus.Debugf("Found forwarding rule: %s", item.Name) + var quota []gcp.QuotaUsage + if item.LoadBalancingScheme == "EXTERNAL" { + quota = []gcp.QuotaUsage{{ + Metric: &gcp.Metric{ + Service: gcp.ServiceComputeEngineAPI, + Limit: "external_network_lb_forwarding_rules", + Dimensions: map[string]string{ + "region": getNameFromURL("regions", item.Region), + }, + }, + Amount: 1, + }} + } + result = append(result, cloudResource{ + key: item.Name, + name: item.Name, + typeName: typeName, + quota: quota, + }) + } } - - return result, nil + return nil } - // Regional forwarding rules - req := o.computeSvc.ForwardingRules.List(o.ProjectID, o.Region).Fields(googleapi.Field(fields)) - if len(filter) > 0 { - req = req.Filter(filter) + var err error + switch typeName { + case globalForwardingRuleResource: + err = o.computeSvc.GlobalForwardingRules.List(o.ProjectID).Fields(googleapi.Field(fields)).Pages(ctx, pagesFunc) + case regionForwardingRuleResource: + err = o.computeSvc.ForwardingRules.List(o.ProjectID, o.Region).Fields(googleapi.Field(fields)).Pages(ctx, pagesFunc) + default: + return nil, fmt.Errorf("invalid forwarding rule type %q", typeName) } - err := req.Pages(ctx, func(list *compute.ForwardingRuleList) error { - result = append(result, createForwardingRuleResources(filterFunc, list)...) - return nil - }) if err != nil { - return nil, fmt.Errorf("failed to list regional forwarding rules: %w", err) + return nil, fmt.Errorf("failed to list forwarding rule: %w", err) } + return result, nil } -func (o *ClusterUninstaller) deleteForwardingRule(ctx context.Context, item cloudResource, scope resourceScope) error { - o.Logger.Debugf("Deleting %s forwarding rule %s", scope, item.name) +func (o *ClusterUninstaller) deleteForwardingRule(ctx context.Context, item cloudResource) error { + o.Logger.Debugf("Deleting forwarding rule %s", item.name) ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() - var op *compute.Operation var err error - if scope == gcpGlobalResource { + var op *compute.Operation + switch item.typeName { + case globalForwardingRuleResource: op, err = o.computeSvc.GlobalForwardingRules.Delete(o.ProjectID, item.name).RequestId(o.requestID(item.typeName, item.name)).Context(ctx).Do() - } else { + case regionForwardingRuleResource: op, err = o.computeSvc.ForwardingRules.Delete(o.ProjectID, o.Region, item.name).RequestId(o.requestID(item.typeName, item.name)).Context(ctx).Do() + default: + return fmt.Errorf("invalid forwarding rule type %q", item.typeName) } if err != nil && !isNoOp(err) { @@ -108,7 +98,7 @@ func (o *ClusterUninstaller) deleteForwardingRule(ctx context.Context, item clou } if op != nil && op.Status == "DONE" && isErrorStatus(op.HttpErrorStatusCode) { o.resetRequestID(item.typeName, item.name) - return fmt.Errorf("failed to delete forwarding rule %s with error: %s: %w", item.name, operationErrorMessage(op), err) + return fmt.Errorf("failed to delete forwarding rule %s with error: %s", item.name, operationErrorMessage(op)) } if op != nil && op.Status == "DONE" { o.resetRequestID(item.typeName, item.name) @@ -121,24 +111,23 @@ func (o *ClusterUninstaller) deleteForwardingRule(ctx context.Context, item clou // destroyForwardingRules removes all forwarding rules with a name prefixed // with the cluster's infra ID. func (o *ClusterUninstaller) destroyForwardingRules(ctx context.Context) error { - for _, scope := range []resourceScope{gcpRegionalResource, gcpGlobalResource} { - found, err := o.listForwardingRules(ctx, scope) + forwardingRuleTypes := []string{globalForwardingRuleResource, regionForwardingRuleResource} + for _, forwardingRuleType := range forwardingRuleTypes { + found, err := o.listForwardingRules(ctx, forwardingRuleType) if err != nil { - return fmt.Errorf("failed to list forwarding rules: %w", err) + return err } - items := o.insertPendingItems("forwardingrule", found) + items := o.insertPendingItems(forwardingRuleType, found) + for _, item := range items { - if err := o.deleteForwardingRule(ctx, item, scope); err != nil { - o.Logger.Errorf("error deleting forwarding rule %s: %w", item.name, err) + err := o.deleteForwardingRule(ctx, item) + if err != nil { o.errorTracker.suppressWarning(item.key, err, o.Logger) } } - if items = o.getPendingItems("forwardingrule"); len(items) > 0 { - for _, item := range items { - if err := o.deleteForwardingRule(ctx, item, scope); err != nil { - return fmt.Errorf("error deleting pending forwarding rule %s: %w", item.name, err) - } - } + + if items := o.getPendingItems(forwardingRuleType); len(items) > 0 { + return fmt.Errorf("%d %s resources pending", len(items), forwardingRuleType) } } return nil diff --git a/pkg/destroy/gcp/gcp.go b/pkg/destroy/gcp/gcp.go index a4c211b8b76..e75f66cbd99 100644 --- a/pkg/destroy/gcp/gcp.go +++ b/pkg/destroy/gcp/gcp.go @@ -34,20 +34,12 @@ var ( longTimeout = 10 * time.Minute ) -type resourceScope string - const ( // capgProviderOwnedLabelFmt is the format string for the label // used for resources created by the Cluster API GCP provider. capgProviderOwnedLabelFmt = "capg-cluster-%s" - // gcpGlobalResource is an identifier to indicate that the resource(s) - // that are being deleted are globally scoped. - gcpGlobalResource resourceScope = "global" - - // gcpRegionalResource is an identifier to indicate that the resource(s) - // that are being deleted are regionally scoped. - gcpRegionalResource resourceScope = "regional" + ownedLabelValue = "owned" ) // ClusterUninstaller holds the various options for the cluster we want to delete @@ -259,8 +251,10 @@ func (o *ClusterUninstaller) clusterIDFilter() string { } func (o *ClusterUninstaller) clusterLabelFilter() string { - return fmt.Sprintf("(labels.%s = \"owned\") OR (labels.%s = \"owned\")", - fmt.Sprintf(gcpconsts.ClusterIDLabelFmt, o.ClusterID), fmt.Sprintf(capgProviderOwnedLabelFmt, o.ClusterID)) + return fmt.Sprintf("(labels.%s = \"%s\") OR (labels.%s = \"%s\")", + fmt.Sprintf(gcpconsts.ClusterIDLabelFmt, o.ClusterID), ownedLabelValue, + fmt.Sprintf(capgProviderOwnedLabelFmt, o.ClusterID), ownedLabelValue, + ) } func (o *ClusterUninstaller) clusterLabelOrClusterIDFilter() string { diff --git a/pkg/destroy/gcp/healthcheck.go b/pkg/destroy/gcp/healthcheck.go index 6c1ba40f102..2f17cddb0d6 100644 --- a/pkg/destroy/gcp/healthcheck.go +++ b/pkg/destroy/gcp/healthcheck.go @@ -4,61 +4,93 @@ import ( "context" "fmt" - "github.com/pkg/errors" "google.golang.org/api/compute/v1" "google.golang.org/api/googleapi" "github.com/openshift/installer/pkg/types/gcp" ) -func (o *ClusterUninstaller) listHealthChecks(ctx context.Context, typeName string, listFunc healthCheckListFunc) ([]cloudResource, error) { - return o.listHealthChecksWithFilter(ctx, typeName, "items(name),nextPageToken", o.clusterIDFilter(), listFunc) +const ( + globalHealthCheckResource = "healthcheck" + regionHealthCheckResource = "regionHealthCheck" +) + +func (o *ClusterUninstaller) listHealthChecks(ctx context.Context, typeName string) ([]cloudResource, error) { + return o.listHealthChecksWithFilter(ctx, typeName, "items(name),nextPageToken", o.isClusterResource) } // listHealthChecksWithFilter lists health checks in the project that satisfy the filter criteria. // The fields parameter specifies which fields should be returned in the result, the filter string contains // a filter string passed to the API to filter results. The filterFunc is a client-side filtering function // that determines whether a particular result should be returned or not. -func (o *ClusterUninstaller) listHealthChecksWithFilter(ctx context.Context, typeName, fields, filter string, listFunc healthCheckListFunc) ([]cloudResource, error) { +func (o *ClusterUninstaller) listHealthChecksWithFilter(ctx context.Context, typeName, fields string, filterFunc resourceFilterFunc) ([]cloudResource, error) { o.Logger.Debugf("Listing health checks") + result := []cloudResource{} + + pagesFunc := func(list *compute.HealthCheckList) error { + for _, item := range list.Items { + if filterFunc(item.Name) { + o.Logger.Debugf("Found health check: %s", item.Name) + result = append(result, cloudResource{ + key: item.Name, + name: item.Name, + typeName: typeName, + quota: []gcp.QuotaUsage{{ + Metric: &gcp.Metric{ + Service: gcp.ServiceComputeEngineAPI, + Limit: "health_checks", + }, + Amount: 1, + }}, + }) + } + } + return nil + } + ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() - result := []cloudResource{} - list, err := listFunc(ctx, filter, fields) + + var err error + switch typeName { + case globalHealthCheckResource: + err = o.computeSvc.HealthChecks.List(o.ProjectID).Fields(googleapi.Field(fields)).Pages(ctx, pagesFunc) + case regionHealthCheckResource: + err = o.computeSvc.RegionHealthChecks.List(o.ProjectID, o.Region).Fields(googleapi.Field(fields)).Pages(ctx, pagesFunc) + default: + return nil, fmt.Errorf("invalid health check type %q", typeName) + } + if err != nil { return nil, fmt.Errorf("failed to list health checks: %w", err) } - for _, item := range list.Items { - o.Logger.Debugf("Found health check: %s", item.Name) - result = append(result, cloudResource{ - key: item.Name, - name: item.Name, - typeName: typeName, - quota: []gcp.QuotaUsage{{ - Metric: &gcp.Metric{ - Service: gcp.ServiceComputeEngineAPI, - Limit: "health_checks", - }, - Amount: 1, - }}, - }) - } return result, nil } -func (o *ClusterUninstaller) deleteHealthCheck(ctx context.Context, item cloudResource, deleteFunc healthCheckDestroyFunc) error { +func (o *ClusterUninstaller) deleteHealthCheck(ctx context.Context, item cloudResource) error { o.Logger.Debugf("Deleting health check %s", item.name) ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() - op, err := deleteFunc(ctx, item) + + var err error + var op *compute.Operation + switch item.typeName { + case globalHealthCheckResource: + op, err = o.computeSvc.HealthChecks.Delete(o.ProjectID, item.name).RequestId(o.requestID(item.typeName, item.name)).Context(ctx).Do() + case regionHealthCheckResource: + op, err = o.computeSvc.RegionHealthChecks.Delete(o.ProjectID, o.Region, item.name).RequestId(o.requestID(item.typeName, item.name)).Context(ctx).Do() + default: + return fmt.Errorf("invalid health check type %q", item.typeName) + } + if err != nil && !isNoOp(err) { o.resetRequestID(item.typeName, item.name) - return errors.Wrapf(err, "failed to delete health check %s", item.name) + return fmt.Errorf("failed to delete health check %s: %w", item.name, err) } if op != nil && op.Status == "DONE" && isErrorStatus(op.HttpErrorStatusCode) { o.resetRequestID(item.typeName, item.name) - return errors.Errorf("failed to delete health check %s with error: %s", item.name, operationErrorMessage(op)) + return fmt.Errorf("failed to delete health check %s with error: %s", item.name, operationErrorMessage(op)) } if (err != nil && isNoOp(err)) || (op != nil && op.Status == "DONE") { o.resetRequestID(item.typeName, item.name) @@ -71,56 +103,25 @@ func (o *ClusterUninstaller) deleteHealthCheck(ctx context.Context, item cloudRe // destroyHealthChecks removes all health check resources that have a name prefixed // with the cluster's infra ID. func (o *ClusterUninstaller) destroyHealthChecks(ctx context.Context) error { - for _, hcd := range []healthCheckDestroyer{ - { - itemTypeName: "healthcheck", - destroyFunc: o.healthCheckDelete, - listFunc: o.healthCheckList, - }, - { - itemTypeName: "regionHealthCheck", - destroyFunc: o.regionHealthCheckDelete, - listFunc: o.regionHealthCheckList, - }, - } { - found, err := o.listHealthChecks(ctx, hcd.itemTypeName, hcd.listFunc) + healthCheckTypes := []string{globalHealthCheckResource, regionHealthCheckResource} + for _, hct := range healthCheckTypes { + found, err := o.listHealthChecks(ctx, hct) if err != nil { return err } - items := o.insertPendingItems(hcd.itemTypeName, found) + items := o.insertPendingItems(hct, found) + for _, item := range items { - err := o.deleteHealthCheck(ctx, item, hcd.destroyFunc) + err := o.deleteHealthCheck(ctx, item) if err != nil { o.errorTracker.suppressWarning(item.key, err, o.Logger) } } - if items = o.getPendingItems(hcd.itemTypeName); len(items) > 0 { - return errors.Errorf("%d items pending", len(items)) + + if items := o.getPendingItems(hct); len(items) > 0 { + return fmt.Errorf("%d %s resources pending", len(items), hct) } } - return nil -} - -type healthCheckListFunc func(ctx context.Context, filter, fields string) (*compute.HealthCheckList, error) -type healthCheckDestroyFunc func(ctx context.Context, item cloudResource) (*compute.Operation, error) -type healthCheckDestroyer struct { - itemTypeName string - destroyFunc healthCheckDestroyFunc - listFunc healthCheckListFunc -} -func (o *ClusterUninstaller) healthCheckDelete(ctx context.Context, item cloudResource) (*compute.Operation, error) { - return o.computeSvc.HealthChecks.Delete(o.ProjectID, item.name).RequestId(o.requestID(item.typeName, item.name)).Context(ctx).Do() -} - -func (o *ClusterUninstaller) healthCheckList(ctx context.Context, filter, fields string) (*compute.HealthCheckList, error) { - return o.computeSvc.HealthChecks.List(o.ProjectID).Filter(filter).Fields(googleapi.Field(fields)).Context(ctx).Do() -} - -func (o *ClusterUninstaller) regionHealthCheckDelete(ctx context.Context, item cloudResource) (*compute.Operation, error) { - return o.computeSvc.RegionHealthChecks.Delete(o.ProjectID, o.Region, item.name).RequestId(o.requestID(item.typeName, item.name)).Context(ctx).Do() -} - -func (o *ClusterUninstaller) regionHealthCheckList(ctx context.Context, filter, fields string) (*compute.HealthCheckList, error) { - return o.computeSvc.RegionHealthChecks.List(o.ProjectID, o.Region).Filter(filter).Fields(googleapi.Field(fields)).Context(ctx).Do() + return nil } diff --git a/pkg/destroy/gcp/httphealthcheck.go b/pkg/destroy/gcp/httphealthcheck.go index 168fbcb990d..cd498fd467f 100644 --- a/pkg/destroy/gcp/httphealthcheck.go +++ b/pkg/destroy/gcp/httphealthcheck.go @@ -10,31 +10,33 @@ import ( "github.com/openshift/installer/pkg/types/gcp" ) +const ( + httpHealthCheckResourceName = "httphealthcheck" +) + func (o *ClusterUninstaller) listHTTPHealthChecks(ctx context.Context) ([]cloudResource, error) { - return o.listHTTPHealthChecksWithFilter(ctx, "items(name),nextPageToken", o.clusterIDFilter(), nil) + return o.listHTTPHealthChecksWithFilter(ctx, "items(name),nextPageToken", o.isClusterResource) } // listHTTPHealthChecksWithFilter lists HTTP Health Checks in the project that satisfy the filter criteria. // The fields parameter specifies which fields should be returned in the result, the filter string contains // a filter string passed to the API to filter results. The filterFunc is a client-side filtering function // that determines whether a particular result should be returned or not. -func (o *ClusterUninstaller) listHTTPHealthChecksWithFilter(ctx context.Context, fields string, filter string, filterFunc func(*compute.HttpHealthCheck) bool) ([]cloudResource, error) { +func (o *ClusterUninstaller) listHTTPHealthChecksWithFilter(ctx context.Context, fields string, filterFunc resourceFilterFunc) ([]cloudResource, error) { o.Logger.Debugf("Listing HTTP health checks") ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() result := []cloudResource{} req := o.computeSvc.HttpHealthChecks.List(o.ProjectID).Fields(googleapi.Field(fields)) - if len(filter) > 0 { - req = req.Filter(filter) - } + err := req.Pages(ctx, func(list *compute.HttpHealthCheckList) error { for _, item := range list.Items { - if filterFunc == nil || filterFunc != nil && filterFunc(item) { + if filterFunc(item.Name) { o.Logger.Debugf("Found HTTP health check: %s", item.Name) result = append(result, cloudResource{ key: item.Name, name: item.Name, - typeName: "httphealthcheck", + typeName: httpHealthCheckResourceName, quota: []gcp.QuotaUsage{{ Metric: &gcp.Metric{ Service: gcp.ServiceComputeEngineAPI, @@ -81,14 +83,14 @@ func (o *ClusterUninstaller) destroyHTTPHealthChecks(ctx context.Context) error if err != nil { return err } - items := o.insertPendingItems("httphealthcheck", found) + items := o.insertPendingItems(httpHealthCheckResourceName, found) for _, item := range items { err := o.deleteHTTPHealthCheck(ctx, item) if err != nil { o.errorTracker.suppressWarning(item.key, err, o.Logger) } } - if items = o.getPendingItems("httphealthcheck"); len(items) > 0 { + if items = o.getPendingItems(httpHealthCheckResourceName); len(items) > 0 { return errors.Errorf("%d items pending", len(items)) } return nil diff --git a/pkg/destroy/gcp/image.go b/pkg/destroy/gcp/image.go index 36d9443a5df..62b87ed764b 100644 --- a/pkg/destroy/gcp/image.go +++ b/pkg/destroy/gcp/image.go @@ -10,31 +10,32 @@ import ( "github.com/openshift/installer/pkg/types/gcp" ) +const ( + imageResourceName = "image" +) + func (o *ClusterUninstaller) listImages(ctx context.Context) ([]cloudResource, error) { - return o.listImagesWithFilter(ctx, "items(name),nextPageToken", o.clusterIDFilter(), nil) + return o.listImagesWithFilter(ctx, "items(name),nextPageToken", o.isClusterResource) } // listImagesWithFilter lists addresses in the project that satisfy the filter criteria. // The fields parameter specifies which fields should be returned in the result, the filter string contains // a filter string passed to the API to filter results. The filterFunc is a client-side filtering function // that determines whether a particular result should be returned or not. -func (o *ClusterUninstaller) listImagesWithFilter(ctx context.Context, fields string, filter string, filterFunc func(*compute.Image) bool) ([]cloudResource, error) { +func (o *ClusterUninstaller) listImagesWithFilter(ctx context.Context, fields string, filterFunc resourceFilterFunc) ([]cloudResource, error) { o.Logger.Debugf("Listing images") ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() result := []cloudResource{} req := o.computeSvc.Images.List(o.ProjectID).Fields(googleapi.Field(fields)) - if len(filter) > 0 { - req = req.Filter(filter) - } err := req.Pages(ctx, func(list *compute.ImageList) error { for _, item := range list.Items { - if filterFunc == nil || filterFunc != nil && filterFunc(item) { + if filterFunc(item.Name) { o.Logger.Debugf("Found image: %s\n", item.Name) result = append(result, cloudResource{ key: item.Name, name: item.Name, - typeName: "image", + typeName: imageResourceName, quota: []gcp.QuotaUsage{{ Metric: &gcp.Metric{ Service: gcp.ServiceComputeEngineAPI, @@ -81,14 +82,14 @@ func (o *ClusterUninstaller) destroyImages(ctx context.Context) error { if err != nil { return err } - items := o.insertPendingItems("image", found) + items := o.insertPendingItems(imageResourceName, found) for _, item := range items { err := o.deleteImage(ctx, item) if err != nil { o.errorTracker.suppressWarning(item.key, err, o.Logger) } } - if items = o.getPendingItems("image"); len(items) > 0 { + if items = o.getPendingItems(imageResourceName); len(items) > 0 { return errors.Errorf("%d items pending", len(items)) } return nil diff --git a/pkg/destroy/gcp/instance.go b/pkg/destroy/gcp/instance.go index 5159fd076aa..19931970756 100644 --- a/pkg/destroy/gcp/instance.go +++ b/pkg/destroy/gcp/instance.go @@ -9,9 +9,17 @@ import ( "google.golang.org/api/compute/v1" "google.golang.org/api/googleapi" + gcpconsts "github.com/openshift/installer/pkg/constants/gcp" "github.com/openshift/installer/pkg/types/gcp" ) +const ( + instanceResourceName = "instance" + stopInstanceResourceName = "stopinstance" +) + +type instanceFilterFunc func(instance *compute.Instance) bool + // getInstanceNameAndZone extracts an instance and zone name from an instance URL in the form: // https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/instances/instance-name // After splitting the service's base path with the work `/projects/`, you get: @@ -27,12 +35,26 @@ func (o *ClusterUninstaller) getInstanceNameAndZone(instanceURL string) (string, } func (o *ClusterUninstaller) listInstances(ctx context.Context) ([]cloudResource, error) { - byName, err := o.listInstancesWithFilter(ctx, "items/*/instances(name,zone,status,machineType),nextPageToken", o.clusterIDFilter(), nil) + byName, err := o.listInstancesWithFilter(ctx, "items/*/instances(name,zone,status,machineType),nextPageToken", + func(item *compute.Instance) bool { + return o.isClusterResource(item.Name) + }) if err != nil { return nil, err } - byLabel, err := o.listInstancesWithFilter(ctx, "items/*/instances(name,zone,status,machineType),nextPageToken", o.clusterLabelFilter(), nil) + byLabel, err := o.listInstancesWithFilter(ctx, "items/*/instances(name,zone,status,machineType),nextPageToken", + func(item *compute.Instance) bool { + for key, value := range item.Labels { + // TODO: do labels get formatted as labels.%s, or is this only for filters + if key == fmt.Sprintf(gcpconsts.ClusterIDLabelFmt, o.ClusterID) && value == ownedLabelValue { + return true + } else if key == fmt.Sprintf(capgProviderOwnedLabelFmt, o.ClusterID) && value == ownedLabelValue { + return true + } + } + return false + }) if err != nil { return nil, err } @@ -43,26 +65,24 @@ func (o *ClusterUninstaller) listInstances(ctx context.Context) ([]cloudResource // The fields parameter specifies which fields should be returned in the result, the filter string contains // a filter string passed to the API to filter results. The filterFunc is a client-side filtering function // that determines whether a particular result should be returned or not. -func (o *ClusterUninstaller) listInstancesWithFilter(ctx context.Context, fields string, filter string, filterFunc func(*compute.Instance) bool) ([]cloudResource, error) { +func (o *ClusterUninstaller) listInstancesWithFilter(ctx context.Context, fields string, filterFunc instanceFilterFunc) ([]cloudResource, error) { o.Logger.Debugf("Listing compute instances") ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() result := []cloudResource{} req := o.computeSvc.Instances.AggregatedList(o.ProjectID).Fields(googleapi.Field(fields)) - if len(filter) > 0 { - req = req.Filter(filter) - } + err := req.Pages(ctx, func(list *compute.InstanceAggregatedList) error { for _, scopedList := range list.Items { for _, item := range scopedList.Instances { - if filterFunc == nil || filterFunc != nil && filterFunc(item) { + if filterFunc(item) { zoneName := o.getZoneName(item.Zone) o.Logger.Debugf("Found instance: %s in zone %s, status %s", item.Name, zoneName, item.Status) result = append(result, cloudResource{ key: fmt.Sprintf("%s/%s", zoneName, item.Name), name: item.Name, status: item.Status, - typeName: "instance", + typeName: instanceResourceName, zone: zoneName, quota: []gcp.QuotaUsage{{ Metric: &gcp.Metric{ @@ -81,7 +101,7 @@ func (o *ClusterUninstaller) listInstancesWithFilter(ctx context.Context, fields return nil }) if err != nil { - return nil, errors.Wrapf(err, "failed to fetch compute instances") + return nil, fmt.Errorf("failed to fetch compute instances: %w", err) } return result, nil } @@ -114,7 +134,7 @@ func (o *ClusterUninstaller) destroyInstances(ctx context.Context) error { if err != nil { return err } - items := o.insertPendingItems("instance", found) + items := o.insertPendingItems(instanceResourceName, found) errs := []error{} for _, item := range items { err := o.deleteInstance(ctx, item) @@ -122,7 +142,7 @@ func (o *ClusterUninstaller) destroyInstances(ctx context.Context) error { errs = append(errs, err) } } - items = o.getPendingItems("instance") + items = o.getPendingItems(instanceResourceName) return aggregateError(errs, len(items)) } @@ -132,21 +152,21 @@ func (o *ClusterUninstaller) stopInstance(ctx context.Context, item cloudResourc defer cancel() op, err := o.computeSvc.Instances. Stop(o.ProjectID, item.zone, item.name). - RequestId(o.requestID("stopinstance", item.zone, item.name)). + RequestId(o.requestID(stopInstanceResourceName, item.zone, item.name)). DiscardLocalSsd(true). Context(ctx). Do() if err != nil && !isNoOp(err) { - o.resetRequestID("stopinstance", item.zone, item.name) + o.resetRequestID(stopInstanceResourceName, item.zone, item.name) return errors.Wrapf(err, "failed to stop instance %s in zone %s", item.name, item.zone) } if op != nil && op.Status == "DONE" && isErrorStatus(op.HttpErrorStatusCode) { - o.resetRequestID("stopinstance", item.zone, item.name) + o.resetRequestID(stopInstanceResourceName, item.zone, item.name) return errors.Errorf("failed to stop instance %s in zone %s with error: %s", item.name, item.zone, operationErrorMessage(op)) } if (err != nil && isNoOp(err)) || (op != nil && op.Status == "DONE") { - o.resetRequestID("stopinstance", item.name) - o.deletePendingItems("stopinstance", []cloudResource{item}) + o.resetRequestID(stopInstanceResourceName, item.name) + o.deletePendingItems(stopInstanceResourceName, []cloudResource{item}) o.Logger.Infof("Stopped instance %s", item.name) } return nil @@ -163,17 +183,17 @@ func (o *ClusterUninstaller) stopInstances(ctx context.Context) error { if item.status != "TERMINATED" { // we record instance quota when we delete the instance, not when we terminate it item.quota = nil - o.insertPendingItems("stopinstance", []cloudResource{item}) + o.insertPendingItems(stopInstanceResourceName, []cloudResource{item}) } } - items := o.getPendingItems("stopinstance") + items := o.getPendingItems(stopInstanceResourceName) for _, item := range items { err := o.stopInstance(ctx, item) if err != nil { o.errorTracker.suppressWarning(item.key, err, o.Logger) } } - if items = o.getPendingItems("stopinstance"); len(items) > 0 { + if items = o.getPendingItems(stopInstanceResourceName); len(items) > 0 { return errors.Errorf("%d items pending", len(items)) } return nil diff --git a/pkg/destroy/gcp/instancegroup.go b/pkg/destroy/gcp/instancegroup.go index 971c0d46b9b..20e1e424845 100644 --- a/pkg/destroy/gcp/instancegroup.go +++ b/pkg/destroy/gcp/instancegroup.go @@ -11,33 +11,34 @@ import ( "github.com/openshift/installer/pkg/types/gcp" ) +const ( + instanceGroupResourceName = "instancegroup" +) + func (o *ClusterUninstaller) listInstanceGroups(ctx context.Context) ([]cloudResource, error) { - return o.listInstanceGroupsWithFilter(ctx, "items/*/instanceGroups(name,selfLink,zone),nextPageToken", o.clusterIDFilter(), nil) + return o.listInstanceGroupsWithFilter(ctx, "items/*/instanceGroups(name,selfLink,zone),nextPageToken", o.isClusterResource) } // listInstanceGroupsWithFilter lists addresses in the project that satisfy the filter criteria. // The fields parameter specifies which fields should be returned in the result, the filter string contains // a filter string passed to the API to filter results. The filterFunc is a client-side filtering function // that determines whether a particular result should be returned or not. -func (o *ClusterUninstaller) listInstanceGroupsWithFilter(ctx context.Context, fields string, filter string, filterFunc func(*compute.InstanceGroup) bool) ([]cloudResource, error) { +func (o *ClusterUninstaller) listInstanceGroupsWithFilter(ctx context.Context, fields string, filterFunc resourceFilterFunc) ([]cloudResource, error) { o.Logger.Debugf("Listing instance groups") ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() result := []cloudResource{} req := o.computeSvc.InstanceGroups.AggregatedList(o.ProjectID).Fields(googleapi.Field(fields)) - if len(filter) > 0 { - req = req.Filter(filter) - } err := req.Pages(ctx, func(list *compute.InstanceGroupAggregatedList) error { for _, scopedList := range list.Items { for _, item := range scopedList.InstanceGroups { - if filterFunc == nil || filterFunc != nil && filterFunc(item) { + if filterFunc(item.Name) { zoneName := o.getZoneName(item.Zone) o.Logger.Debugf("Found instance group: %s in zone %s", item.Name, zoneName) result = append(result, cloudResource{ key: fmt.Sprintf("%s/%s", zoneName, item.Name), name: item.Name, - typeName: "instancegroup", + typeName: instanceGroupResourceName, zone: zoneName, url: item.SelfLink, quota: []gcp.QuotaUsage{{ @@ -72,7 +73,7 @@ func (o *ClusterUninstaller) deleteInstanceGroup(ctx context.Context, item cloud return errors.Wrapf(err, "failed to delete instance group %s in zone %s", item.name, item.zone) } if op != nil && op.Status == "DONE" && isErrorStatus(op.HttpErrorStatusCode) { - o.resetRequestID("instancegroup", item.zone, item.name) + o.resetRequestID(instanceGroupResourceName, item.zone, item.name) return errors.Errorf("failed to delete instance group %s in zone %s with error: %s", item.name, item.zone, operationErrorMessage(op)) } if (err != nil && isNoOp(err)) || (op != nil && op.Status == "DONE") { @@ -90,14 +91,14 @@ func (o *ClusterUninstaller) destroyInstanceGroups(ctx context.Context) error { if err != nil { return err } - items := o.insertPendingItems("instancegroup", found) + items := o.insertPendingItems(instanceGroupResourceName, found) for _, item := range items { err := o.deleteInstanceGroup(ctx, item) if err != nil { o.errorTracker.suppressWarning(item.key, err, o.Logger) } } - if items = o.getPendingItems("instancegroup"); len(items) > 0 { + if items = o.getPendingItems(instanceGroupResourceName); len(items) > 0 { return errors.Errorf("%d items pending", len(items)) } return nil diff --git a/pkg/destroy/gcp/network.go b/pkg/destroy/gcp/network.go index cd2cc9b3434..fa19bc87a3b 100644 --- a/pkg/destroy/gcp/network.go +++ b/pkg/destroy/gcp/network.go @@ -2,37 +2,40 @@ package gcp import ( "context" + "fmt" "github.com/pkg/errors" compute "google.golang.org/api/compute/v1" "google.golang.org/api/googleapi" ) +const ( + networkResourceName = "network" +) + func (o *ClusterUninstaller) listNetworks(ctx context.Context) ([]cloudResource, error) { - return o.listNetworksWithFilter(ctx, "items(name,selfLink),nextPageToken", o.clusterIDFilter(), nil) + return o.listNetworksWithFilter(ctx, "items(name,selfLink),nextPageToken") } // listNetworksWithFilter lists addresses in the project that satisfy the filter criteria. // The fields parameter specifies which fields should be returned in the result, the filter string contains // a filter string passed to the API to filter results. The filterFunc is a client-side filtering function // that determines whether a particular result should be returned or not. -func (o *ClusterUninstaller) listNetworksWithFilter(ctx context.Context, fields string, filter string, filterFunc func(*compute.Network) bool) ([]cloudResource, error) { +func (o *ClusterUninstaller) listNetworksWithFilter(ctx context.Context, fields string) ([]cloudResource, error) { o.Logger.Debugf("Listing networks") ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() result := []cloudResource{} req := o.computeSvc.Networks.List(o.ProjectID).Fields(googleapi.Field(fields)) - if len(filter) > 0 { - req = req.Filter(filter) - } + err := req.Pages(ctx, func(list *compute.NetworkList) error { for _, item := range list.Items { - if filterFunc == nil || filterFunc != nil && filterFunc(item) { + if o.isClusterResource(item.Name) { o.Logger.Debugf("Found network: %s", item.Name) result = append(result, cloudResource{ key: item.Name, name: item.Name, - typeName: "network", + typeName: networkResourceName, url: item.SelfLink, }) } @@ -40,7 +43,7 @@ func (o *ClusterUninstaller) listNetworksWithFilter(ctx context.Context, fields return nil }) if err != nil { - return nil, errors.Wrapf(err, "failed to list networks") + return nil, fmt.Errorf("failed to list networks: %w", err) } return result, nil } @@ -73,14 +76,14 @@ func (o *ClusterUninstaller) destroyNetworks(ctx context.Context) error { if err != nil { return err } - items := o.insertPendingItems("network", found) + items := o.insertPendingItems(networkResourceName, found) for _, item := range items { foundRoutes, err := o.listNetworkRoutes(ctx, item.url) if err != nil { o.errorTracker.suppressWarning(item.key, err, o.Logger) continue } - routes := o.insertPendingItems("route", foundRoutes) + routes := o.insertPendingItems(routeResourceName, foundRoutes) for _, route := range routes { err := o.deleteRoute(ctx, route) if err != nil { @@ -92,7 +95,7 @@ func (o *ClusterUninstaller) destroyNetworks(ctx context.Context) error { o.errorTracker.suppressWarning(item.key, err, o.Logger) } } - if items = o.getPendingItems("network"); len(items) > 0 { + if items = o.getPendingItems(networkResourceName); len(items) > 0 { return errors.Errorf("%d items pending", len(items)) } return nil diff --git a/pkg/destroy/gcp/route.go b/pkg/destroy/gcp/route.go index afce74547f3..b5748c13c13 100644 --- a/pkg/destroy/gcp/route.go +++ b/pkg/destroy/gcp/route.go @@ -12,30 +12,34 @@ import ( "github.com/openshift/installer/pkg/types/gcp" ) +const ( + routeResourceName = "resource" +) + +type routeFilterFunc func(item *compute.Route) bool + func (o *ClusterUninstaller) listNetworkRoutes(ctx context.Context, networkURL string) ([]cloudResource, error) { - return o.listRoutesWithFilter(ctx, "items(name),nextPageToken", fmt.Sprintf("network eq %q", networkURL), nil) + return o.listRoutesWithFilter(ctx, "items(name),nextPageToken", func(item *compute.Route) bool { return item.Network == networkURL }) } func (o *ClusterUninstaller) listRoutes(ctx context.Context) ([]cloudResource, error) { - return o.listRoutesWithFilter(ctx, "items(name),nextPageToken", o.clusterIDFilter(), nil) + return o.listRoutesWithFilter(ctx, "items(name),nextPageToken", func(item *compute.Route) bool { return o.isClusterResource(item.Name) }) } // listRoutesWithFilter lists routes in the project that satisfy the filter criteria. // The fields parameter specifies which fields should be returned in the result, the filter string contains // a filter string passed to the API to filter results. The filterFunc is a client-side filtering function // that determines whether a particular result should be returned or not. -func (o *ClusterUninstaller) listRoutesWithFilter(ctx context.Context, fields string, filter string, filterFunc func(*compute.Route) bool) ([]cloudResource, error) { +func (o *ClusterUninstaller) listRoutesWithFilter(ctx context.Context, fields string, filterFunc routeFilterFunc) ([]cloudResource, error) { o.Logger.Debugf("Listing routes") ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() result := []cloudResource{} req := o.computeSvc.Routes.List(o.ProjectID).Fields(googleapi.Field(fields)) - if len(filter) > 0 { - req = req.Filter(filter) - } + err := req.Pages(ctx, func(list *compute.RouteList) error { for _, item := range list.Items { - if filterFunc == nil || filterFunc != nil && filterFunc(item) { + if filterFunc(item) { if strings.HasPrefix(item.Name, "default-route-") { continue } @@ -43,7 +47,7 @@ func (o *ClusterUninstaller) listRoutesWithFilter(ctx context.Context, fields st result = append(result, cloudResource{ key: item.Name, name: item.Name, - typeName: "route", + typeName: routeResourceName, quota: []gcp.QuotaUsage{{ Metric: &gcp.Metric{ Service: gcp.ServiceComputeEngineAPI, @@ -57,7 +61,7 @@ func (o *ClusterUninstaller) listRoutesWithFilter(ctx context.Context, fields st return nil }) if err != nil { - return nil, errors.Wrapf(err, "failed to list routes") + return nil, fmt.Errorf("failed to list routes: %w", err) } return result, nil } @@ -90,14 +94,14 @@ func (o *ClusterUninstaller) destroyRoutes(ctx context.Context) error { if err != nil { return err } - items := o.insertPendingItems("route", found) + items := o.insertPendingItems(routeResourceName, found) for _, item := range items { err := o.deleteRoute(ctx, item) if err != nil { o.errorTracker.suppressWarning(item.key, err, o.Logger) } } - if items = o.getPendingItems("route"); len(items) > 0 { + if items = o.getPendingItems(routeResourceName); len(items) > 0 { return errors.Errorf("%d items pending", len(items)) } return nil diff --git a/pkg/destroy/gcp/router.go b/pkg/destroy/gcp/router.go index c841a2ddd14..4a7dd419314 100644 --- a/pkg/destroy/gcp/router.go +++ b/pkg/destroy/gcp/router.go @@ -10,31 +10,32 @@ import ( "github.com/openshift/installer/pkg/types/gcp" ) +const ( + routerResourceName = "router" +) + func (o *ClusterUninstaller) listRouters(ctx context.Context) ([]cloudResource, error) { - return o.listRoutersWithFilter(ctx, "items(name),nextPageToken", o.clusterIDFilter(), nil) + return o.listRoutersWithFilter(ctx, "items(name),nextPageToken", o.isClusterResource) } // listRoutersWithFilter lists routers in the project that satisfy the filter criteria. // The fields parameter specifies which fields should be returned in the result, the filter string contains // a filter string passed to the API to filter results. The filterFunc is a client-side filtering function // that determines whether a particular result should be returned or not. -func (o *ClusterUninstaller) listRoutersWithFilter(ctx context.Context, fields string, filter string, filterFunc func(*compute.Router) bool) ([]cloudResource, error) { +func (o *ClusterUninstaller) listRoutersWithFilter(ctx context.Context, fields string, filterFunc resourceFilterFunc) ([]cloudResource, error) { o.Logger.Debug("Listing routers") ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() result := []cloudResource{} req := o.computeSvc.Routers.List(o.ProjectID, o.Region).Fields(googleapi.Field(fields)) - if len(filter) > 0 { - req = req.Filter(filter) - } err := req.Pages(ctx, func(list *compute.RouterList) error { for _, item := range list.Items { - if filterFunc == nil || filterFunc != nil && filterFunc(item) { + if filterFunc(item.Name) { o.Logger.Debugf("Found router: %s", item.Name) result = append(result, cloudResource{ key: item.Name, name: item.Name, - typeName: "router", + typeName: routerResourceName, quota: []gcp.QuotaUsage{{ Metric: &gcp.Metric{ Service: gcp.ServiceComputeEngineAPI, @@ -81,14 +82,14 @@ func (o *ClusterUninstaller) destroyRouters(ctx context.Context) error { if err != nil { return err } - items := o.insertPendingItems("router", found) + items := o.insertPendingItems(routerResourceName, found) for _, item := range items { err := o.deleteRouter(ctx, item) if err != nil { o.errorTracker.suppressWarning(item.key, err, o.Logger) } } - if items = o.getPendingItems("router"); len(items) > 0 { + if items = o.getPendingItems(routerResourceName); len(items) > 0 { return errors.Errorf("%d items pending", len(items)) } return nil diff --git a/pkg/destroy/gcp/subnetwork.go b/pkg/destroy/gcp/subnetwork.go index e248ddee5d4..579763bf8c4 100644 --- a/pkg/destroy/gcp/subnetwork.go +++ b/pkg/destroy/gcp/subnetwork.go @@ -10,31 +10,32 @@ import ( "github.com/openshift/installer/pkg/types/gcp" ) +const ( + subnetworkResourceName = "subnetwork" +) + func (o *ClusterUninstaller) listSubnetworks(ctx context.Context) ([]cloudResource, error) { - return o.listSubnetworksWithFilter(ctx, "items(name,network),nextPageToken", o.clusterIDFilter(), nil) + return o.listSubnetworksWithFilter(ctx, "items(name,network),nextPageToken", o.isClusterResource) } // listSubnetworksWithFilter lists subnetworks in the project that satisfy the filter criteria. // The fields parameter specifies which fields should be returned in the result, the filter string contains // a filter string passed to the API to filter results. The filterFunc is a client-side filtering function // that determines whether a particular result should be returned or not. -func (o *ClusterUninstaller) listSubnetworksWithFilter(ctx context.Context, fields string, filter string, filterFunc func(*compute.Subnetwork) bool) ([]cloudResource, error) { +func (o *ClusterUninstaller) listSubnetworksWithFilter(ctx context.Context, fields string, filterFunc resourceFilterFunc) ([]cloudResource, error) { o.Logger.Debugf("Listing subnetworks") ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() result := []cloudResource{} req := o.computeSvc.Subnetworks.List(o.ProjectID, o.Region).Fields(googleapi.Field(fields)) - if len(filter) > 0 { - req = req.Filter(filter) - } err := req.Pages(ctx, func(list *compute.SubnetworkList) error { for _, item := range list.Items { - if filterFunc == nil || filterFunc != nil && filterFunc(item) { + if filterFunc(item.Name) { o.Logger.Debugf("Found subnetwork: %s", item.Name) result = append(result, cloudResource{ key: item.Name, name: item.Name, - typeName: "subnetwork", + typeName: subnetworkResourceName, quota: []gcp.QuotaUsage{{ Metric: &gcp.Metric{ Service: gcp.ServiceComputeEngineAPI, @@ -90,14 +91,14 @@ func (o *ClusterUninstaller) destroySubnetworks(ctx context.Context) error { if err != nil { return err } - items := o.insertPendingItems("subnetwork", found) + items := o.insertPendingItems(subnetworkResourceName, found) for _, item := range items { err := o.deleteSubnetwork(ctx, item) if err != nil { o.errorTracker.suppressWarning(item.key, err, o.Logger) } } - if items = o.getPendingItems("subnetwork"); len(items) > 0 { + if items = o.getPendingItems(subnetworkResourceName); len(items) > 0 { return errors.Errorf("%d items pending", len(items)) } return nil diff --git a/pkg/destroy/gcp/targetTCPProxies.go b/pkg/destroy/gcp/targetTCPProxies.go index e26f9f80479..74691666093 100644 --- a/pkg/destroy/gcp/targetTCPProxies.go +++ b/pkg/destroy/gcp/targetTCPProxies.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "google.golang.org/api/compute/v1" "google.golang.org/api/googleapi" "github.com/openshift/installer/pkg/types/gcp" @@ -14,35 +15,41 @@ const ( ) func (o *ClusterUninstaller) listTargetTCPProxies(ctx context.Context, typeName string) ([]cloudResource, error) { - return o.listTargetTCPProxiesWithFilter(ctx, typeName, "items(name),nextPageToken", o.clusterIDFilter()) + return o.listTargetTCPProxiesWithFilter(ctx, typeName, "items(name),nextPageToken", o.isClusterResource) } // listTargetTCPProxiesWithFilter lists target TCP Proxies in the project that satisfy the filter criteria. -func (o *ClusterUninstaller) listTargetTCPProxiesWithFilter(ctx context.Context, typeName, fields, filter string) ([]cloudResource, error) { +func (o *ClusterUninstaller) listTargetTCPProxiesWithFilter(ctx context.Context, typeName, fields string, filterFunc resourceFilterFunc) ([]cloudResource, error) { o.Logger.Debugf("Listing target tcp proxies") ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() - list, err := o.computeSvc.TargetTcpProxies.List(o.ProjectID).Filter(filter).Fields(googleapi.Field(fields)).Context(ctx).Do() - if err != nil { - return nil, fmt.Errorf("failed to list target tcp proxies: %w", err) + result := []cloudResource{} + + pagesFunc := func(list *compute.TargetTcpProxyList) error { + for _, item := range list.Items { + if filterFunc(item.Name) { + o.Logger.Debugf("Found target TCP proxy: %s", item.Name) + result = append(result, cloudResource{ + key: item.Name, + name: item.Name, + typeName: typeName, + quota: []gcp.QuotaUsage{{ + Metric: &gcp.Metric{ + Service: gcp.ServiceComputeEngineAPI, + Limit: "target_tcp_proxy", + }, + Amount: 1, + }}, + }) + } + } + return nil } - result := []cloudResource{} - for _, item := range list.Items { - o.Logger.Debugf("Found target TCP proxy: %s", item.Name) - result = append(result, cloudResource{ - key: item.Name, - name: item.Name, - typeName: typeName, - quota: []gcp.QuotaUsage{{ - Metric: &gcp.Metric{ - Service: gcp.ServiceComputeEngineAPI, - Limit: "target_tcp_proxy", - }, - Amount: 1, - }}, - }) + err := o.computeSvc.TargetTcpProxies.List(o.ProjectID).Fields(googleapi.Field(fields)).Pages(ctx, pagesFunc) + if err != nil { + return nil, fmt.Errorf("failed to list target tcp proxies: %w", err) } return result, nil diff --git a/pkg/destroy/gcp/targetpool.go b/pkg/destroy/gcp/targetpool.go index cfeec6486c0..b3b6ff75855 100644 --- a/pkg/destroy/gcp/targetpool.go +++ b/pkg/destroy/gcp/targetpool.go @@ -10,31 +10,37 @@ import ( "github.com/openshift/installer/pkg/types/gcp" ) +const ( + targetPoolResourceName = "targetpool" +) + +type targetPoolFilterFunc func(pool *compute.TargetPool) bool + func (o *ClusterUninstaller) listTargetPools(ctx context.Context) ([]cloudResource, error) { - return o.listTargetPoolsWithFilter(ctx, "items(name),nextPageToken", o.clusterIDFilter(), nil) + return o.listTargetPoolsWithFilter(ctx, "items(name),nextPageToken", func(item *compute.TargetPool) bool { + return o.isClusterResource(item.Name) + }) } // listTargetPoolsWithFilter lists target pools in the project that satisfy the filter criteria. // The fields parameter specifies which fields should be returned in the result, the filter string contains // a filter string passed to the API to filter results. The filterFunc is a client-side filtering function // that determines whether a particular result should be returned or not. -func (o *ClusterUninstaller) listTargetPoolsWithFilter(ctx context.Context, fields string, filter string, filterFunc func(*compute.TargetPool) bool) ([]cloudResource, error) { +func (o *ClusterUninstaller) listTargetPoolsWithFilter(ctx context.Context, fields string, filterFunc targetPoolFilterFunc) ([]cloudResource, error) { o.Logger.Debugf("Listing target pools") ctx, cancel := context.WithTimeout(ctx, defaultTimeout) defer cancel() result := []cloudResource{} req := o.computeSvc.TargetPools.List(o.ProjectID, o.Region).Fields(googleapi.Field(fields)) - if len(filter) > 0 { - req = req.Filter(filter) - } + err := req.Pages(ctx, func(list *compute.TargetPoolList) error { for _, item := range list.Items { - if filterFunc == nil || filterFunc != nil && filterFunc(item) { + if filterFunc(item) { o.Logger.Debugf("Found target pool: %s", item.Name) result = append(result, cloudResource{ key: item.Name, name: item.Name, - typeName: "targetpool", + typeName: targetPoolResourceName, quota: []gcp.QuotaUsage{{ Metric: &gcp.Metric{ Service: gcp.ServiceComputeEngineAPI, @@ -81,14 +87,14 @@ func (o *ClusterUninstaller) destroyTargetPools(ctx context.Context) error { if err != nil { return err } - items := o.insertPendingItems("targetpool", found) + items := o.insertPendingItems(targetPoolResourceName, found) for _, item := range items { err := o.deleteTargetPool(ctx, item) if err != nil { o.errorTracker.suppressWarning(item.key, err, o.Logger) } } - if items = o.getPendingItems("targetpool"); len(items) > 0 { + if items = o.getPendingItems(targetPoolResourceName); len(items) > 0 { return errors.Errorf("%d items pending", len(items)) } return nil