Add retry on cloud functions error when pulling source from GCS (#3570)

third_party/terraform/resources/resource_cloudfunctions_function.go

@@ -401,21 +401,27 @@ func resourceCloudFunctionsCreate(d *schema.ResourceData, meta interface{}) erro
 	}
 
 	log.Printf("[DEBUG] Creating cloud function: %s", function.Name)
-	op, err := config.clientCloudFunctions.Projects.Locations.Functions.Create(
-		cloudFuncId.locationId(), function).Do()
-	if err != nil {
-		return err
-	}
 
-	// Name of function should be unique
-	d.SetId(cloudFuncId.cloudFunctionId())
+	// We retry the whole create-and-wait because Cloud Functions
+	// will sometimes fail a creation operation entirely if it fails to pull
+	// source code and we need to try the whole creation again.
+	rerr := retryTimeDuration(func() error {
+		op, err := config.clientCloudFunctions.Projects.Locations.Functions.Create(
+			cloudFuncId.locationId(), function).Do()
+		if err != nil {
+			return err
+		}
+
+		// Name of function should be unique
+		d.SetId(cloudFuncId.cloudFunctionId())
 
-	err = cloudFunctionsOperationWait(config, op, "Creating CloudFunctions Function",
-		d.Timeout(schema.TimeoutCreate))
-	if err != nil {
-		return err
+		return cloudFunctionsOperationWait(config, op, "Creating CloudFunctions Function",
+			d.Timeout(schema.TimeoutCreate))
+	}, d.Timeout(schema.TimeoutCreate), isCloudFunctionsSourceCodeError)
+	if rerr != nil {
+		return rerr
 	}
-
+	log.Printf("[DEBUG] Finished creating cloud function: %s", function.Name)
 	return resourceCloudFunctionsRead(d, meta)
 }
 

third_party/terraform/utils/common_operation.go

@@ -9,6 +9,15 @@ import (
 	cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1"
 )
 
+// Wraps Op.Error in an implementation of built-in Error
+type CommonOpError struct {
+	*cloudresourcemanager.Status
+}
+
+func (e *CommonOpError) Error() string {
+	return fmt.Sprintf("Error code %v, message: %s", e.Code, e.Message)
+}
+
 type Waiter interface {
 	// State returns the current status of the operation.
 	State() string
@@ -56,7 +65,7 @@ func (w *CommonOperationWaiter) State() string {
 
 func (w *CommonOperationWaiter) Error() error {
 	if w != nil && w.Op.Error != nil {
-		return fmt.Errorf("Error code %v, message: %s", w.Op.Error.Code, w.Op.Error.Message)
+		return &CommonOpError{w.Op.Error}
 	}
 	return nil
 }

third_party/terraform/utils/error_retry_predicates.go

@@ -257,3 +257,12 @@ func isPeeringOperationInProgress(err error) (bool, string) {
 	}
 	return false, ""
 }
+
+func isCloudFunctionsSourceCodeError(err error) (bool, string) {
+	if operr, ok := err.(*CommonOpError); ok {
+		if operr.Code == 3 && operr.Message == "Failed to retrieve function source code" {
+			return true, fmt.Sprintf("Retry on Function failing to pull code from GCS")
+		}
+	}
+	return false, ""
+}