Retry SQL delete calls on 409 (#1169)

Merged PR #1169.
GoogleCloudPlatform · Jan 8, 2019 · 52ec03f · 52ec03f
1 parent e5baf7d
commit 52ec03f
Show file tree

Hide file tree

Showing 6 changed files with 20 additions and 10 deletions.
diff --git a/build/terraform b/build/terraform
diff --git a/build/terraform-beta b/build/terraform-beta
diff --git a/third_party/terraform/resources/resource_sql_database.go b/third_party/terraform/resources/resource_sql_database.go
@@ -7,7 +7,7 @@ import (
 
 	"github.com/hashicorp/terraform/helper/schema"
 
-	"google.golang.org/api/sqladmin/v1beta4"
+	sqladmin "google.golang.org/api/sqladmin/v1beta4"
 )
 
 func resourceSqlDatabase() *schema.Resource {
@@ -206,10 +206,10 @@ func resourceSqlDatabaseDelete(d *schema.ResourceData, meta interface{}) error {
 	defer mutexKV.Unlock(instanceMutexKey(project, instance_name))
 
 	var op *sqladmin.Operation
-	err = retryTime(func() error {
+	err = retryTimeDuration(func() error {
 		op, err = config.clientSqlAdmin.Databases.Delete(project, instance_name, database_name).Do()
 		return err
-	}, 5 /* minutes */)
+	}, d.Timeout(schema.TimeoutDelete))
 
 	if err != nil {
 		return fmt.Errorf("Error, failed to delete"+

diff --git a/third_party/terraform/resources/resource_sql_database_instance.go.erb b/third_party/terraform/resources/resource_sql_database_instance.go.erb
@@ -14,7 +14,7 @@ import (
 	"github.com/hashicorp/terraform/helper/validation"
 
 	"google.golang.org/api/googleapi"
-	"google.golang.org/api/sqladmin/v1beta4"
+	sqladmin "google.golang.org/api/sqladmin/v1beta4"
 )
 
 const privateNetworkLinkRegex = "projects/(" + ProjectRegex + ")/global/networks/((?:[a-z](?:[-a-z0-9]*[a-z0-9])?))$"
@@ -783,7 +783,11 @@ func resourceSqlDatabaseInstanceDelete(d *schema.ResourceData, meta interface{})
 		defer mutexKV.Unlock(instanceMutexKey(project, v.(string)))
 	}
 
-	op, err := config.clientSqlAdmin.Instances.Delete(project, d.Get("name").(string)).Do()
+	var op *sqladmin.Operation
+	err = retryTimeDuration(func() error {
+		op, err = config.clientSqlAdmin.Instances.Delete(project, d.Get("name").(string)).Do()
+		return err
+	}, d.Timeout(schema.TimeoutDelete))
 
 	if err != nil {
 		return fmt.Errorf("Error, failed to delete instance %s: %s", d.Get("name").(string), err)

diff --git a/third_party/terraform/resources/resource_sql_user.go b/third_party/terraform/resources/resource_sql_user.go
@@ -6,7 +6,7 @@ import (
 	"strings"
 
 	"github.com/hashicorp/terraform/helper/schema"
-	"google.golang.org/api/sqladmin/v1beta4"
+	sqladmin "google.golang.org/api/sqladmin/v1beta4"
 )
 
 func resourceSqlUser() *schema.Resource {
@@ -206,7 +206,12 @@ func resourceSqlUserDelete(d *schema.ResourceData, meta interface{}) error {
 
 	mutexKV.Lock(instanceMutexKey(project, instance))
 	defer mutexKV.Unlock(instanceMutexKey(project, instance))
-	op, err := config.clientSqlAdmin.Users.Delete(project, instance, host, name).Do()
+
+	var op *sqladmin.Operation
+	err = retryTimeDuration(func() error {
+		op, err = config.clientSqlAdmin.Users.Delete(project, instance, host, name).Do()
+		return err
+	}, d.Timeout(schema.TimeoutDelete))
 
 	if err != nil {
 		return fmt.Errorf("Error, failed to delete"+

diff --git a/third_party/terraform/utils/utils.go b/third_party/terraform/utils/utils.go
@@ -331,7 +331,8 @@ func retryTimeDuration(retryFunc func() error, duration time.Duration) error {
 }
 
 func isRetryableError(err error) bool {
-	if gerr, ok := err.(*googleapi.Error); ok && (gerr.Code == 429 || gerr.Code == 500 || gerr.Code == 502 || gerr.Code == 503) {
+	// 409's are retried because cloud sql throws a 409 when concurrent calls are made
+	if gerr, ok := err.(*googleapi.Error); ok && (gerr.Code == 409 || gerr.Code == 429 || gerr.Code == 500 || gerr.Code == 502 || gerr.Code == 503) {
 		return true
 	}
 	return false