Retry delete calls on 409

GoogleCloudPlatform · Jan 5, 2019 · 5ffcdf5 · 5ffcdf5
1 parent e339329
commit 5ffcdf5
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 5 deletions.
diff --git a/third_party/terraform/resources/resource_sql_database_instance.go.erb b/third_party/terraform/resources/resource_sql_database_instance.go.erb
@@ -14,7 +14,7 @@ import (
 	"github.com/hashicorp/terraform/helper/validation"
 
 	"google.golang.org/api/googleapi"
-	"google.golang.org/api/sqladmin/v1beta4"
+	sqladmin "google.golang.org/api/sqladmin/v1beta4"
 )
 
 const privateNetworkLinkRegex = "projects/(" + ProjectRegex + ")/global/networks/((?:[a-z](?:[-a-z0-9]*[a-z0-9])?))$"
@@ -783,7 +783,11 @@ func resourceSqlDatabaseInstanceDelete(d *schema.ResourceData, meta interface{})
 		defer mutexKV.Unlock(instanceMutexKey(project, v.(string)))
 	}
 
-	op, err := config.clientSqlAdmin.Instances.Delete(project, d.Get("name").(string)).Do()
+	var op *sqladmin.Operation
+	err = retryTime(func() error {
+		op, err = config.clientSqlAdmin.Instances.Delete(project, d.Get("name").(string)).Do()
+		return err
+	}, 5)
 
 	if err != nil {
 		return fmt.Errorf("Error, failed to delete instance %s: %s", d.Get("name").(string), err)

diff --git a/third_party/terraform/resources/resource_sql_user.go b/third_party/terraform/resources/resource_sql_user.go
@@ -6,7 +6,7 @@ import (
 	"strings"
 
 	"github.com/hashicorp/terraform/helper/schema"
-	"google.golang.org/api/sqladmin/v1beta4"
+	sqladmin "google.golang.org/api/sqladmin/v1beta4"
 )
 
 func resourceSqlUser() *schema.Resource {
@@ -206,7 +206,12 @@ func resourceSqlUserDelete(d *schema.ResourceData, meta interface{}) error {
 
 	mutexKV.Lock(instanceMutexKey(project, instance))
 	defer mutexKV.Unlock(instanceMutexKey(project, instance))
-	op, err := config.clientSqlAdmin.Users.Delete(project, instance, host, name).Do()
+
+	var op *sqladmin.Operation
+	err = retryTime(func() error {
+		op, err = config.clientSqlAdmin.Users.Delete(project, instance, host, name).Do()
+		return err
+	}, 5 /* minutes */)
 
 	if err != nil {
 		return fmt.Errorf("Error, failed to delete"+

diff --git a/third_party/terraform/utils/utils.go b/third_party/terraform/utils/utils.go
@@ -331,7 +331,8 @@ func retryTimeDuration(retryFunc func() error, duration time.Duration) error {
 }
 
 func isRetryableError(err error) bool {
-	if gerr, ok := err.(*googleapi.Error); ok && (gerr.Code == 429 || gerr.Code == 500 || gerr.Code == 502 || gerr.Code == 503) {
+	// 409's are retried because cloud sql throws a 409 when concurrent calls are made
+	if gerr, ok := err.(*googleapi.Error); ok && (gerr.Code == 409 || gerr.Code == 429 || gerr.Code == 500 || gerr.Code == 502 || gerr.Code == 503) {
 		return true
 	}
 	return false