From 0d17df773297189bd63fd0333ff2821af731e979 Mon Sep 17 00:00:00 2001 From: Scott Suarez Date: Fri, 17 Sep 2021 13:59:00 -0700 Subject: [PATCH 1/8] retry on service usage 403 and support billing_project --- .../resources/resource_google_project.go | 30 ++++++++++++++----- .../resource_google_project_service.go | 17 +++++++++-- .../terraform/utils/error_retry_predicates.go | 14 +++++++++ .../terraform/utils/retry_utils.go | 13 ++++++++ .../terraform/utils/serviceusage_batching.go | 22 ++++++++++---- 5 files changed, 81 insertions(+), 15 deletions(-) diff --git a/mmv1/third_party/terraform/resources/resource_google_project.go b/mmv1/third_party/terraform/resources/resource_google_project.go index 38aa6477b7f8..2d1e38b4bbab 100644 --- a/mmv1/third_party/terraform/resources/resource_google_project.go +++ b/mmv1/third_party/terraform/resources/resource_google_project.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log" + "net/http" "regexp" "strconv" "strings" @@ -17,6 +18,11 @@ import ( "google.golang.org/api/serviceusage/v1" ) +type ServicesCall interface { + Header() http.Header + Do(opts ...googleapi.CallOption) (*serviceusage.Operation, error) +} + // resourceGoogleProject returns a *schema.Resource that allows a customer // to declare a Google Cloud Project resource. func resourceGoogleProject() *schema.Resource { @@ -179,7 +185,7 @@ func resourceGoogleProjectCreate(d *schema.ResourceData, meta interface{}) error // a network and deleting it in the background. if !d.Get("auto_create_network").(bool) { // The compute API has to be enabled before we can delete a network. - if err = enableServiceUsageProjectServices([]string{"compute.googleapis.com"}, project.ProjectId, userAgent, config, d.Timeout(schema.TimeoutCreate)); err != nil { + if err = enableServiceUsageProjectServices([]string{"compute.googleapis.com"}, project.ProjectId, "", userAgent, config, d.Timeout(schema.TimeoutCreate)); err != nil { return errwrap.Wrapf("Error enabling the Compute Engine API required to delete the default network: {{err}} ", err) } @@ -580,7 +586,7 @@ func readGoogleProject(d *schema.ResourceData, config *Config, userAgent string) } // Enables services. WARNING: Use BatchRequestEnableServices for better batching if possible. -func enableServiceUsageProjectServices(services []string, project, userAgent string, config *Config, timeout time.Duration) error { +func enableServiceUsageProjectServices(services []string, project, billingProject, userAgent string, config *Config, timeout time.Duration) error { // ServiceUsage does not allow more than 20 services to be enabled per // batchEnable API call. See // https://cloud.google.com/service-usage/docs/reference/rest/v1/services/batchEnable @@ -595,7 +601,7 @@ func enableServiceUsageProjectServices(services []string, project, userAgent str return nil } - if err := doEnableServicesRequest(nextBatch, project, userAgent, config, timeout); err != nil { + if err := doEnableServicesRequest(nextBatch, project, billingProject, userAgent, config, timeout); err != nil { return err } log.Printf("[DEBUG] Finished enabling next batch of %d project services: %+v", len(nextBatch), nextBatch) @@ -605,9 +611,9 @@ func enableServiceUsageProjectServices(services []string, project, userAgent str return waitForServiceUsageEnabledServices(services, project, userAgent, config, timeout) } -func doEnableServicesRequest(services []string, project, userAgent string, config *Config, timeout time.Duration) error { +func doEnableServicesRequest(services []string, project, billingProject, userAgent string, config *Config, timeout time.Duration) error { var op *serviceusage.Operation - + var call ServicesCall err := retryTimeDuration(func() error { var rerr error if len(services) == 1 { @@ -615,15 +621,23 @@ func doEnableServicesRequest(services []string, project, userAgent string, confi // using service endpoint. name := fmt.Sprintf("projects/%s/services/%s", project, services[0]) req := &serviceusage.EnableServiceRequest{} - op, rerr = config.NewServiceUsageClient(userAgent).Services.Enable(name, req).Do() + call = config.NewServiceUsageClient(userAgent).Services.Enable(name, req) } else { // Batch enable for multiple services. name := fmt.Sprintf("projects/%s", project) req := &serviceusage.BatchEnableServicesRequest{ServiceIds: services} - op, rerr = config.NewServiceUsageClient(userAgent).Services.BatchEnable(name, req).Do() + call = config.NewServiceUsageClient(userAgent).Services.BatchEnable(name, req) + } + if config.UserProjectOverride && billingProject != "" { + call.Header().Add("X-Goog-User-Project", billingProject) } + op, rerr = call.Do() return handleServiceUsageRetryableError(rerr) - }, timeout, serviceUsageServiceBeingActivated) + }, + timeout, + serviceUsageServiceBeingActivated, + retryOn403NTimes(4), // retry 4 times due to self referential activation taking time to propagate tpg#9489 + ) if err != nil { return errwrap.Wrapf("failed to send enable services request: {{err}}", err) } diff --git a/mmv1/third_party/terraform/resources/resource_google_project_service.go b/mmv1/third_party/terraform/resources/resource_google_project_service.go index 4341c98a2912..e2e673304652 100644 --- a/mmv1/third_party/terraform/resources/resource_google_project_service.go +++ b/mmv1/third_party/terraform/resources/resource_google_project_service.go @@ -187,7 +187,13 @@ func resourceGoogleProjectServiceRead(d *schema.ResourceData, meta interface{}) // Verify project for services still exists projectGetCall := config.NewResourceManagerClient(userAgent).Projects.Get(project) if config.UserProjectOverride { - projectGetCall.Header().Add("X-Goog-User-Project", project) + billingProject := project + + // err == nil indicates that the billing_project value was found + if bp, err := getBillingProject(d, config); err == nil { + billingProject = bp + } + projectGetCall.Header().Add("X-Goog-User-Project", billingProject) } p, err := projectGetCall.Do() @@ -268,7 +274,14 @@ func disableServiceUsageProjectService(service, project string, d *schema.Resour DisableDependentServices: disableDependentServices, }) if config.UserProjectOverride { - servicesDisableCall.Header().Add("X-Goog-User-Project", project) + billingProject := project + + // err == nil indicates that the billing_project value was found + if bp, err := getBillingProject(d, config); err == nil { + billingProject = bp + } + + servicesDisableCall.Header().Add("X-Goog-User-Project", billingProject) } sop, err := servicesDisableCall.Do() if err != nil { diff --git a/mmv1/third_party/terraform/utils/error_retry_predicates.go b/mmv1/third_party/terraform/utils/error_retry_predicates.go index 8f4c933310e4..b69e0ff3c594 100644 --- a/mmv1/third_party/terraform/utils/error_retry_predicates.go +++ b/mmv1/third_party/terraform/utils/error_retry_predicates.go @@ -206,6 +206,20 @@ func serviceUsageServiceBeingActivated(err error) (bool, string) { return false, "" } +func retryOn403NTimes(n int) func(error) (bool, string) { + count := 0 + + return func(err error) (bool, string) { + if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 403 { + if count < n { + count += 1 + return true, fmt.Sprintf("retrying on 403 %s more times", count-n-1) + } + } + return false, "" + } +} + // Retry if Bigquery operation returns a 403 with a specific message for // concurrent operations (which are implemented in terms of 'edit quota'). func isBigqueryIAMQuotaError(err error) (bool, string) { diff --git a/mmv1/third_party/terraform/utils/retry_utils.go b/mmv1/third_party/terraform/utils/retry_utils.go index 81517ec6196e..89280df05870 100644 --- a/mmv1/third_party/terraform/utils/retry_utils.go +++ b/mmv1/third_party/terraform/utils/retry_utils.go @@ -29,6 +29,19 @@ func retryTimeDuration(retryFunc func() error, duration time.Duration, errorRetr }) } +func retryTimeDurationWithPrecondition(retryFunc func() error, duration time.Duration, errorRetryPredicates ...RetryErrorPredicateFunc) error { + return resource.Retry(duration, func() *resource.RetryError { + err := retryFunc() + if err == nil { + return nil + } + if isRetryableError(err, errorRetryPredicates...) { + return resource.RetryableError(err) + } + return resource.NonRetryableError(err) + }) +} + func isRetryableError(topErr error, customPredicates ...RetryErrorPredicateFunc) bool { if topErr == nil { return false diff --git a/mmv1/third_party/terraform/utils/serviceusage_batching.go b/mmv1/third_party/terraform/utils/serviceusage_batching.go index 9504e82153a9..b5cbcce877e6 100644 --- a/mmv1/third_party/terraform/utils/serviceusage_batching.go +++ b/mmv1/third_party/terraform/utils/serviceusage_batching.go @@ -27,11 +27,17 @@ func BatchRequestEnableService(service string, project string, d *schema.Resourc return err } + billingProject := project + // err == nil indicates that the billing_project value was found + if bp, err := getBillingProject(d, config); err == nil { + billingProject = bp + } + req := &BatchRequest{ ResourceName: project, Body: []string{service}, CombineF: combineServiceUsageServicesBatches, - SendF: sendBatchFuncEnableServices(config, userAgent, d.Timeout(schema.TimeoutCreate)), + SendF: sendBatchFuncEnableServices(config, userAgent, billingProject, d.Timeout(schema.TimeoutCreate)), DebugId: fmt.Sprintf("Enable Project Service %q for project %q", service, project), } @@ -51,11 +57,17 @@ func tryEnableRenamedService(service, altName string, project string, d *schema. log.Printf("[DEBUG] found renamed service %s (with alternate name %s)", service, altName) // use a short timeout- failures are likely + billingProject := project + // err == nil indicates that the billing_project value was found + if bp, err := getBillingProject(d, config); err == nil { + billingProject = bp + } + log.Printf("[DEBUG] attempting enabling service with user-specified name %s", service) - err = enableServiceUsageProjectServices([]string{service}, project, userAgent, config, 1*time.Minute) + err = enableServiceUsageProjectServices([]string{service}, project, billingProject, userAgent, config, 1*time.Minute) if err != nil { log.Printf("[DEBUG] saw error %s. attempting alternate name %v", err, altName) - err2 := enableServiceUsageProjectServices([]string{altName}, project, userAgent, config, 1*time.Minute) + err2 := enableServiceUsageProjectServices([]string{altName}, project, billingProject, userAgent, config, 1*time.Minute) if err2 != nil { return fmt.Errorf("Saw 2 subsequent errors attempting to enable a renamed service: %s / %s", err, err2) } @@ -97,13 +109,13 @@ func combineServiceUsageServicesBatches(srvsRaw interface{}, toAddRaw interface{ return append(srvs, toAdd...), nil } -func sendBatchFuncEnableServices(config *Config, userAgent string, timeout time.Duration) BatcherSendFunc { +func sendBatchFuncEnableServices(config *Config, userAgent, billingProject string, timeout time.Duration) BatcherSendFunc { return func(project string, toEnableRaw interface{}) (interface{}, error) { toEnable, ok := toEnableRaw.([]string) if !ok { return nil, fmt.Errorf("Expected batch body type to be []string, got %v. This is a provider error.", toEnableRaw) } - return nil, enableServiceUsageProjectServices(toEnable, project, userAgent, config, timeout) + return nil, enableServiceUsageProjectServices(toEnable, project, billingProject, userAgent, config, timeout) } } From 136bb105a2087e657a7b1fdb21d411b842c6072d Mon Sep 17 00:00:00 2001 From: Scott Suarez Date: Fri, 17 Sep 2021 14:00:43 -0700 Subject: [PATCH 2/8] fix linting errors --- .../terraform/utils/error_retry_predicates.go | 2 +- mmv1/third_party/terraform/utils/retry_utils.go | 13 ------------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/mmv1/third_party/terraform/utils/error_retry_predicates.go b/mmv1/third_party/terraform/utils/error_retry_predicates.go index b69e0ff3c594..3ab995c5fb1d 100644 --- a/mmv1/third_party/terraform/utils/error_retry_predicates.go +++ b/mmv1/third_party/terraform/utils/error_retry_predicates.go @@ -213,7 +213,7 @@ func retryOn403NTimes(n int) func(error) (bool, string) { if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 403 { if count < n { count += 1 - return true, fmt.Sprintf("retrying on 403 %s more times", count-n-1) + return true, fmt.Sprintf("retrying on 403 %v more times", count-n-1) } } return false, "" diff --git a/mmv1/third_party/terraform/utils/retry_utils.go b/mmv1/third_party/terraform/utils/retry_utils.go index 89280df05870..81517ec6196e 100644 --- a/mmv1/third_party/terraform/utils/retry_utils.go +++ b/mmv1/third_party/terraform/utils/retry_utils.go @@ -29,19 +29,6 @@ func retryTimeDuration(retryFunc func() error, duration time.Duration, errorRetr }) } -func retryTimeDurationWithPrecondition(retryFunc func() error, duration time.Duration, errorRetryPredicates ...RetryErrorPredicateFunc) error { - return resource.Retry(duration, func() *resource.RetryError { - err := retryFunc() - if err == nil { - return nil - } - if isRetryableError(err, errorRetryPredicates...) { - return resource.RetryableError(err) - } - return resource.NonRetryableError(err) - }) -} - func isRetryableError(topErr error, customPredicates ...RetryErrorPredicateFunc) bool { if topErr == nil { return false From 906bb7daecb3476e0e6fbcb60157d9b0e7cdddb4 Mon Sep 17 00:00:00 2001 From: Scott Suarez Date: Fri, 17 Sep 2021 14:09:22 -0700 Subject: [PATCH 3/8] set billing project for project creation service call --- .../terraform/resources/resource_google_project.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mmv1/third_party/terraform/resources/resource_google_project.go b/mmv1/third_party/terraform/resources/resource_google_project.go index 2d1e38b4bbab..3463a41af60f 100644 --- a/mmv1/third_party/terraform/resources/resource_google_project.go +++ b/mmv1/third_party/terraform/resources/resource_google_project.go @@ -185,7 +185,14 @@ func resourceGoogleProjectCreate(d *schema.ResourceData, meta interface{}) error // a network and deleting it in the background. if !d.Get("auto_create_network").(bool) { // The compute API has to be enabled before we can delete a network. - if err = enableServiceUsageProjectServices([]string{"compute.googleapis.com"}, project.ProjectId, "", userAgent, config, d.Timeout(schema.TimeoutCreate)); err != nil { + + billingProject := ProjectId + // err == nil indicates that the billing_project value was found + if bp, err := getBillingProject(d, config); err == nil { + billingProject = bp + } + + if err = enableServiceUsageProjectServices([]string{"compute.googleapis.com"}, project.ProjectId, billingProject, userAgent, config, d.Timeout(schema.TimeoutCreate)); err != nil { return errwrap.Wrapf("Error enabling the Compute Engine API required to delete the default network: {{err}} ", err) } From d2fed446098356efca3e800d355783cb82758af9 Mon Sep 17 00:00:00 2001 From: Scott Suarez Date: Fri, 17 Sep 2021 14:09:59 -0700 Subject: [PATCH 4/8] lint error fix --- mmv1/third_party/terraform/resources/resource_google_project.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmv1/third_party/terraform/resources/resource_google_project.go b/mmv1/third_party/terraform/resources/resource_google_project.go index 3463a41af60f..211a15ead40e 100644 --- a/mmv1/third_party/terraform/resources/resource_google_project.go +++ b/mmv1/third_party/terraform/resources/resource_google_project.go @@ -186,7 +186,7 @@ func resourceGoogleProjectCreate(d *schema.ResourceData, meta interface{}) error if !d.Get("auto_create_network").(bool) { // The compute API has to be enabled before we can delete a network. - billingProject := ProjectId + billingProject := project.ProjectId // err == nil indicates that the billing_project value was found if bp, err := getBillingProject(d, config); err == nil { billingProject = bp From bdfa723059d3b787adf62be8e0b5baf21e2ba438 Mon Sep 17 00:00:00 2001 From: Scott Suarez Date: Tue, 21 Sep 2021 11:48:51 -0700 Subject: [PATCH 5/8] specify guard to be more narrow --- mmv1/third_party/terraform/utils/error_retry_predicates.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mmv1/third_party/terraform/utils/error_retry_predicates.go b/mmv1/third_party/terraform/utils/error_retry_predicates.go index 3ab995c5fb1d..4a395c92d826 100644 --- a/mmv1/third_party/terraform/utils/error_retry_predicates.go +++ b/mmv1/third_party/terraform/utils/error_retry_predicates.go @@ -206,12 +206,13 @@ func serviceUsageServiceBeingActivated(err error) (bool, string) { return false, "" } +// Retries errors on 403 n number of times if the error message +// returned contains `has not been used in project` func retryOn403NTimes(n int) func(error) (bool, string) { count := 0 - return func(err error) (bool, string) { if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 403 { - if count < n { + if count < n && strings.Contains(gerr.Body, "has not been used in project") { count += 1 return true, fmt.Sprintf("retrying on 403 %v more times", count-n-1) } From 864db4b4e4bec6befcedec766819802831d7ed1a Mon Sep 17 00:00:00 2001 From: Scott Suarez Date: Tue, 21 Sep 2021 13:59:25 -0700 Subject: [PATCH 6/8] remove retry code --- .../resources/resource_google_project.go | 3 +-- .../terraform/utils/error_retry_predicates.go | 15 --------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/mmv1/third_party/terraform/resources/resource_google_project.go b/mmv1/third_party/terraform/resources/resource_google_project.go index 211a15ead40e..8432297141d1 100644 --- a/mmv1/third_party/terraform/resources/resource_google_project.go +++ b/mmv1/third_party/terraform/resources/resource_google_project.go @@ -642,8 +642,7 @@ func doEnableServicesRequest(services []string, project, billingProject, userAge return handleServiceUsageRetryableError(rerr) }, timeout, - serviceUsageServiceBeingActivated, - retryOn403NTimes(4), // retry 4 times due to self referential activation taking time to propagate tpg#9489 + serviceUsageServiceBeingActivated ) if err != nil { return errwrap.Wrapf("failed to send enable services request: {{err}}", err) diff --git a/mmv1/third_party/terraform/utils/error_retry_predicates.go b/mmv1/third_party/terraform/utils/error_retry_predicates.go index 4a395c92d826..8f4c933310e4 100644 --- a/mmv1/third_party/terraform/utils/error_retry_predicates.go +++ b/mmv1/third_party/terraform/utils/error_retry_predicates.go @@ -206,21 +206,6 @@ func serviceUsageServiceBeingActivated(err error) (bool, string) { return false, "" } -// Retries errors on 403 n number of times if the error message -// returned contains `has not been used in project` -func retryOn403NTimes(n int) func(error) (bool, string) { - count := 0 - return func(err error) (bool, string) { - if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 403 { - if count < n && strings.Contains(gerr.Body, "has not been used in project") { - count += 1 - return true, fmt.Sprintf("retrying on 403 %v more times", count-n-1) - } - } - return false, "" - } -} - // Retry if Bigquery operation returns a 403 with a specific message for // concurrent operations (which are implemented in terms of 'edit quota'). func isBigqueryIAMQuotaError(err error) (bool, string) { From 0917be88725a1125776349b396ae77820a240568 Mon Sep 17 00:00:00 2001 From: Scott Suarez Date: Wed, 22 Sep 2021 17:14:54 -0700 Subject: [PATCH 7/8] update all reference to add billing project --- .../resources/resource_google_project.go | 27 ++++++++++++------- .../resource_google_project_service.go | 2 -- .../resource_google_project_service_test.go | 3 +-- .../terraform/utils/serviceusage_batching.go | 12 ++++++--- 4 files changed, 27 insertions(+), 17 deletions(-) diff --git a/mmv1/third_party/terraform/resources/resource_google_project.go b/mmv1/third_party/terraform/resources/resource_google_project.go index 8432297141d1..0c2f6a44d987 100644 --- a/mmv1/third_party/terraform/resources/resource_google_project.go +++ b/mmv1/third_party/terraform/resources/resource_google_project.go @@ -225,7 +225,13 @@ func resourceGoogleProjectCheckPreRequisites(config *Config, d *schema.ResourceD return fmt.Errorf("missing permission on %q: %v", ba, perm) } if !d.Get("auto_create_network").(bool) { - _, err := config.NewServiceUsageClient(userAgent).Services.Get("projects/00000000000/services/serviceusage.googleapis.com").Do() + call := config.NewServiceUsageClient(userAgent).Services.Get("projects/00000000000/services/serviceusage.googleapis.com") + if config.UserProjectOverride { + if billingProject, err := getBillingProject(d, config); err == nil { + call.Header().Add("X-Goog-User-Project", billingProject) + } + } + _, err := call.Do() switch { // We are querying a dummy project since the call is already coming from the quota project. // If the API is enabled we get a not found message or accessNotConfigured if API is not enabled. @@ -615,7 +621,7 @@ func enableServiceUsageProjectServices(services []string, project, billingProjec } log.Printf("[DEBUG] Verifying that all services are enabled") - return waitForServiceUsageEnabledServices(services, project, userAgent, config, timeout) + return waitForServiceUsageEnabledServices(services, project, billingProject, userAgent, config, timeout) } func doEnableServicesRequest(services []string, project, billingProject, userAgent string, config *Config, timeout time.Duration) error { @@ -642,7 +648,7 @@ func doEnableServicesRequest(services []string, project, billingProject, userAge return handleServiceUsageRetryableError(rerr) }, timeout, - serviceUsageServiceBeingActivated + serviceUsageServiceBeingActivated, ) if err != nil { return errwrap.Wrapf("failed to send enable services request: {{err}}", err) @@ -659,15 +665,16 @@ func doEnableServicesRequest(services []string, project, billingProject, userAge // if a service has been renamed, this function will list both the old and new // forms of the service. LIST responses are expected to return only the old or // new form, but we'll always return both. -func listCurrentlyEnabledServices(project, userAgent string, config *Config, timeout time.Duration) (map[string]struct{}, error) { +func listCurrentlyEnabledServices(project, billingProject, userAgent string, config *Config, timeout time.Duration) (map[string]struct{}, error) { log.Printf("[DEBUG] Listing enabled services for project %s", project) apiServices := make(map[string]struct{}) err := retryTimeDuration(func() error { ctx := context.Background() - return config.NewServiceUsageClient(userAgent).Services. - List(fmt.Sprintf("projects/%s", project)). - Fields("services/name,nextPageToken"). - Filter("state:ENABLED"). + call := config.NewServiceUsageClient(userAgent).Services.List(fmt.Sprintf("projects/%s", project)) + if config.UserProjectOverride && billingProject != "" { + call.Header().Add("X-Goog-User-Project", billingProject) + } + return call.Fields("services/name,nextPageToken").Filter("state:ENABLED"). Pages(ctx, func(r *serviceusage.ListServicesResponse) error { for _, v := range r.Services { // services are returned as "projects/{{project}}/services/{{name}}" @@ -697,13 +704,13 @@ func listCurrentlyEnabledServices(project, userAgent string, config *Config, tim // waitForServiceUsageEnabledServices doesn't resend enable requests - it just // waits for service enablement status to propagate. Essentially, it waits until // all services show up as enabled when listing services on the project. -func waitForServiceUsageEnabledServices(services []string, project, userAgent string, config *Config, timeout time.Duration) error { +func waitForServiceUsageEnabledServices(services []string, project, billingProject, userAgent string, config *Config, timeout time.Duration) error { missing := make([]string, 0, len(services)) delay := time.Duration(0) interval := time.Second err := retryTimeDuration(func() error { // Get the list of services that are enabled on the project - enabledServices, err := listCurrentlyEnabledServices(project, userAgent, config, timeout) + enabledServices, err := listCurrentlyEnabledServices(project, billingProject, userAgent, config, timeout) if err != nil { return err } diff --git a/mmv1/third_party/terraform/resources/resource_google_project_service.go b/mmv1/third_party/terraform/resources/resource_google_project_service.go index e2e673304652..0ec58d6d9418 100644 --- a/mmv1/third_party/terraform/resources/resource_google_project_service.go +++ b/mmv1/third_party/terraform/resources/resource_google_project_service.go @@ -275,12 +275,10 @@ func disableServiceUsageProjectService(service, project string, d *schema.Resour }) if config.UserProjectOverride { billingProject := project - // err == nil indicates that the billing_project value was found if bp, err := getBillingProject(d, config); err == nil { billingProject = bp } - servicesDisableCall.Header().Add("X-Goog-User-Project", billingProject) } sop, err := servicesDisableCall.Do() diff --git a/mmv1/third_party/terraform/tests/resource_google_project_service_test.go b/mmv1/third_party/terraform/tests/resource_google_project_service_test.go index 7d5719cb7301..0842440e7c66 100644 --- a/mmv1/third_party/terraform/tests/resource_google_project_service_test.go +++ b/mmv1/third_party/terraform/tests/resource_google_project_service_test.go @@ -198,8 +198,7 @@ func TestAccProjectService_renamedService(t *testing.T) { func testAccCheckProjectService(t *testing.T, services []string, pid string, expectEnabled bool) resource.TestCheckFunc { return func(s *terraform.State) error { config := googleProviderConfig(t) - - currentlyEnabled, err := listCurrentlyEnabledServices(pid, config.userAgent, config, time.Minute*10) + currentlyEnabled, err := listCurrentlyEnabledServices(pid, "", config.userAgent, config, time.Minute*10) if err != nil { return fmt.Errorf("Error listing services for project %q: %v", pid, err) } diff --git a/mmv1/third_party/terraform/utils/serviceusage_batching.go b/mmv1/third_party/terraform/utils/serviceusage_batching.go index b5cbcce877e6..8bff91790cac 100644 --- a/mmv1/third_party/terraform/utils/serviceusage_batching.go +++ b/mmv1/third_party/terraform/utils/serviceusage_batching.go @@ -81,12 +81,18 @@ func BatchRequestReadServices(project string, d *schema.ResourceData, config *Co return nil, err } + billingProject := project + // err == nil indicates that the billing_project value was found + if bp, err := getBillingProject(d, config); err == nil { + billingProject = bp + } + req := &BatchRequest{ ResourceName: project, Body: nil, // Use empty CombineF since the request is exactly the same no matter how many services we read. CombineF: func(body interface{}, toAdd interface{}) (interface{}, error) { return nil, nil }, - SendF: sendListServices(config, userAgent, d.Timeout(schema.TimeoutRead)), + SendF: sendListServices(config, billingProject, userAgent, d.Timeout(schema.TimeoutRead)), DebugId: fmt.Sprintf("List Project Services %s", project), } @@ -119,8 +125,8 @@ func sendBatchFuncEnableServices(config *Config, userAgent, billingProject strin } } -func sendListServices(config *Config, userAgent string, timeout time.Duration) BatcherSendFunc { +func sendListServices(config *Config, billingProject, userAgent string, timeout time.Duration) BatcherSendFunc { return func(project string, _ interface{}) (interface{}, error) { - return listCurrentlyEnabledServices(project, userAgent, config, timeout) + return listCurrentlyEnabledServices(project, billingProject, userAgent, config, timeout) } } From 13211a13199c2eaff1d95a20942a8bda406e4b3a Mon Sep 17 00:00:00 2001 From: Scott Suarez Date: Tue, 12 Oct 2021 12:35:58 -0700 Subject: [PATCH 8/8] use billing project with polling operation --- .../terraform/resources/resource_google_project.go | 2 +- .../terraform/resources/resource_google_project_service.go | 5 ++--- .../resources/resource_project_service_identity.go.erb | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/mmv1/third_party/terraform/resources/resource_google_project.go b/mmv1/third_party/terraform/resources/resource_google_project.go index 0c2f6a44d987..182c8d4f2007 100644 --- a/mmv1/third_party/terraform/resources/resource_google_project.go +++ b/mmv1/third_party/terraform/resources/resource_google_project.go @@ -654,7 +654,7 @@ func doEnableServicesRequest(services []string, project, billingProject, userAge return errwrap.Wrapf("failed to send enable services request: {{err}}", err) } // Poll for the API to return - waitErr := serviceUsageOperationWait(config, op, project, fmt.Sprintf("Enable Project %q Services: %+v", project, services), userAgent, timeout) + waitErr := serviceUsageOperationWait(config, op, billingProject, fmt.Sprintf("Enable Project %q Services: %+v", project, services), userAgent, timeout) if waitErr != nil { return waitErr } diff --git a/mmv1/third_party/terraform/resources/resource_google_project_service.go b/mmv1/third_party/terraform/resources/resource_google_project_service.go index 0ec58d6d9418..9de6b268d20e 100644 --- a/mmv1/third_party/terraform/resources/resource_google_project_service.go +++ b/mmv1/third_party/terraform/resources/resource_google_project_service.go @@ -264,17 +264,16 @@ func resourceGoogleProjectServiceUpdate(d *schema.ResourceData, meta interface{} // Disables a project service. func disableServiceUsageProjectService(service, project string, d *schema.ResourceData, config *Config, disableDependentServices bool) error { err := retryTimeDuration(func() error { + billingProject := project userAgent, err := generateUserAgentString(d, config.userAgent) if err != nil { return err } - name := fmt.Sprintf("projects/%s/services/%s", project, service) servicesDisableCall := config.NewServiceUsageClient(userAgent).Services.Disable(name, &serviceusage.DisableServiceRequest{ DisableDependentServices: disableDependentServices, }) if config.UserProjectOverride { - billingProject := project // err == nil indicates that the billing_project value was found if bp, err := getBillingProject(d, config); err == nil { billingProject = bp @@ -286,7 +285,7 @@ func disableServiceUsageProjectService(service, project string, d *schema.Resour return err } // Wait for the operation to complete - waitErr := serviceUsageOperationWait(config, sop, project, "api to disable", userAgent, d.Timeout(schema.TimeoutDelete)) + waitErr := serviceUsageOperationWait(config, sop, billingProject, "api to disable", userAgent, d.Timeout(schema.TimeoutDelete)) if waitErr != nil { return waitErr } diff --git a/mmv1/third_party/terraform/resources/resource_project_service_identity.go.erb b/mmv1/third_party/terraform/resources/resource_project_service_identity.go.erb index 14bcec0f4624..5bd1a7b42a02 100644 --- a/mmv1/third_party/terraform/resources/resource_project_service_identity.go.erb +++ b/mmv1/third_party/terraform/resources/resource_project_service_identity.go.erb @@ -74,7 +74,7 @@ func resourceProjectServiceIdentityCreate(d *schema.ResourceData, meta interface var opRes map[string]interface{} err = serviceUsageOperationWaitTimeWithResponse( - config, res, &opRes, project, "Creating Service Identity", userAgent, + config, res, &opRes, billingProject, "Creating Service Identity", userAgent, d.Timeout(schema.TimeoutCreate)) if err != nil { return err