retry node pool writes on failed precondition (#1660)

Hypothetically fixes #1643.

@thomasriley, are you able to patch this change into your provider to see if it fixed the problem? I haven't been able to get a working repo so I haven't verified the fix yet.

google/resource_container_node_pool.go

@@ -218,20 +218,34 @@ func resourceContainerNodePoolCreate(d *schema.ResourceData, meta interface{}) e
 		NodePool: nodePool,
 	}
 
-	operation, err := config.clientContainerBeta.
-		Projects.Locations.Clusters.NodePools.Create(nodePoolInfo.parent(), req).Do()
+	timeout := d.Timeout(schema.TimeoutCreate)
+	startTime := time.Now()
 
+	var operation *containerBeta.Operation
+	err = resource.Retry(timeout, func() *resource.RetryError {
+		operation, err = config.clientContainerBeta.
+			Projects.Locations.Clusters.NodePools.Create(nodePoolInfo.parent(), req).Do()
+
+		if err != nil {
+			if isFailedPreconditionError(err) {
+				// We get failed precondition errors if the cluster is updating
+				// while we try to add the node pool.
+				return resource.RetryableError(err)
+			}
+			return resource.NonRetryableError(err)
+		}
+		return nil
+	})
 	if err != nil {
 		return fmt.Errorf("error creating NodePool: %s", err)
 	}
+	timeout -= time.Since(startTime)
 
 	d.SetId(fmt.Sprintf("%s/%s/%s", nodePoolInfo.location, nodePoolInfo.cluster, nodePool.Name))
 
-	timeoutInMinutes := int(d.Timeout(schema.TimeoutCreate).Minutes())
-
 	waitErr := containerBetaOperationWait(config,
 		operation, nodePoolInfo.project,
-		nodePoolInfo.location, "creating GKE NodePool", timeoutInMinutes, 3)
+		nodePoolInfo.location, "creating GKE NodePool", int(timeout.Minutes()), 3)
 
 	if waitErr != nil {
 		// The resource didn't actually create
@@ -259,10 +273,6 @@ func resourceContainerNodePoolRead(d *schema.ResourceData, meta interface{}) err
 		nodePool, err = config.clientContainerBeta.
 			Projects.Locations.Clusters.NodePools.Get(nodePoolInfo.fullyQualifiedName(name)).Do()
 
-		if err != nil {
-			return resource.NonRetryableError(err)
-		}
-
 		if err != nil {
 			return resource.NonRetryableError(err)
 		}

google/utils.go

@@ -166,6 +166,25 @@ func isApiNotEnabledError(err error) bool {
 	return false
 }
 
+func isFailedPreconditionError(err error) bool {
+	gerr, ok := errwrap.GetType(err, &googleapi.Error{}).(*googleapi.Error)
+	if !ok {
+		return false
+	}
+	if gerr == nil {
+		return false
+	}
+	if gerr.Code != 400 {
+		return false
+	}
+	for _, e := range gerr.Errors {
+		if e.Reason == "failedPrecondition" {
+			return true
+		}
+	}
+	return false
+}
+
 func isConflictError(err error) bool {
 	if e, ok := err.(*googleapi.Error); ok && e.Code == 409 {
 		return true