resource/aws_lb_target_group: Handle read-after-create eventual consi…

…stency (#18634)

* resource/aws_lb_target_group: Handle read-after-create eventual consistency

Reference: #16796
Reference: #18633

Output from acceptance testing in AWS Commercial:

```
--- PASS: TestAccAWSLBTargetGroup_BackwardsCompatibility (59.15s)
--- PASS: TestAccAWSLBTargetGroup_basic (58.91s)
--- PASS: TestAccAWSLBTargetGroup_basicUdp (30.43s)
--- PASS: TestAccAWSLBTargetGroup_changeNameForceNew (89.34s)
--- PASS: TestAccAWSLBTargetGroup_changePortForceNew (87.16s)
--- PASS: TestAccAWSLBTargetGroup_changeProtocolForceNew (94.14s)
--- PASS: TestAccAWSLBTargetGroup_changeVpcForceNew (72.33s)
--- PASS: TestAccAWSLBTargetGroup_defaults_application (54.39s)
--- PASS: TestAccAWSLBTargetGroup_defaults_network (44.14s)
--- PASS: TestAccAWSLBTargetGroup_enableHealthCheck (46.41s)
--- PASS: TestAccAWSLBTargetGroup_generatedName (42.89s)
--- PASS: TestAccAWSLBTargetGroup_namePrefix (42.98s)
--- PASS: TestAccAWSLBTargetGroup_networkLB_TargetGroup (90.36s)
--- PASS: TestAccAWSLBTargetGroup_networkLB_TargetGroupWithProxy (83.68s)
--- PASS: TestAccAWSLBTargetGroup_preserveClientIPValid (70.93s)
--- PASS: TestAccAWSLBTargetGroup_Protocol_Geneve (26.71s)
--- PASS: TestAccAWSLBTargetGroup_Protocol_Tcp_HealthCheck_Protocol (62.56s)
--- PASS: TestAccAWSLBTargetGroup_Protocol_Tls (32.36s)
--- PASS: TestAccAWSLBTargetGroup_ProtocolVersion (35.54s)
--- PASS: TestAccAWSLBTargetGroup_ProtocolVersion_GRPC_HealthCheck (38.35s)
--- PASS: TestAccAWSLBTargetGroup_ProtocolVersion_HTTP_GRPC_Update (80.57s)
--- PASS: TestAccAWSLBTargetGroup_stickinessDefaultALB (42.96s)
--- PASS: TestAccAWSLBTargetGroup_stickinessDefaultNLB (107.85s)
--- PASS: TestAccAWSLBTargetGroup_stickinessInvalidALB (66.41s)
--- PASS: TestAccAWSLBTargetGroup_stickinessInvalidNLB (49.40s)
--- PASS: TestAccAWSLBTargetGroup_stickinessValidALB (83.32s)
--- PASS: TestAccAWSLBTargetGroup_stickinessValidNLB (156.84s)
--- PASS: TestAccAWSLBTargetGroup_tags (91.23s)
--- PASS: TestAccAWSLBTargetGroup_TCP_HTTPHealthCheck (85.65s)
--- PASS: TestAccAWSLBTargetGroup_updateHealthCheck (84.79s)
--- PASS: TestAccAWSLBTargetGroup_updateSticknessEnabled (92.92s)
--- PASS: TestAccAWSLBTargetGroup_withoutHealthcheck (25.39s)
```

Output from acceptance testing in AWS GovCloud (US):

```
--- PASS: TestAccAWSLBTargetGroup_BackwardsCompatibility (47.26s)
--- PASS: TestAccAWSLBTargetGroup_basic (56.74s)
--- PASS: TestAccAWSLBTargetGroup_basicUdp (40.39s)
--- PASS: TestAccAWSLBTargetGroup_changeNameForceNew (90.81s)
--- PASS: TestAccAWSLBTargetGroup_changePortForceNew (97.77s)
--- PASS: TestAccAWSLBTargetGroup_changeProtocolForceNew (90.81s)
--- PASS: TestAccAWSLBTargetGroup_changeVpcForceNew (87.07s)
--- PASS: TestAccAWSLBTargetGroup_defaults_application (33.70s)
--- PASS: TestAccAWSLBTargetGroup_defaults_network (56.92s)
--- PASS: TestAccAWSLBTargetGroup_enableHealthCheck (71.70s)
--- PASS: TestAccAWSLBTargetGroup_generatedName (46.95s)
--- PASS: TestAccAWSLBTargetGroup_namePrefix (56.98s)
--- PASS: TestAccAWSLBTargetGroup_networkLB_TargetGroup (135.01s)
--- PASS: TestAccAWSLBTargetGroup_networkLB_TargetGroupWithProxy (87.42s)
--- PASS: TestAccAWSLBTargetGroup_Protocol_Tcp_HealthCheck_Protocol (90.85s)
--- PASS: TestAccAWSLBTargetGroup_Protocol_Tls (41.97s)
--- PASS: TestAccAWSLBTargetGroup_ProtocolVersion (43.90s)
--- PASS: TestAccAWSLBTargetGroup_ProtocolVersion_GRPC_HealthCheck (57.01s)
--- PASS: TestAccAWSLBTargetGroup_ProtocolVersion_HTTP_GRPC_Update (84.70s)
--- PASS: TestAccAWSLBTargetGroup_stickinessDefaultALB (45.57s)
--- PASS: TestAccAWSLBTargetGroup_stickinessDefaultNLB (129.50s)
--- PASS: TestAccAWSLBTargetGroup_stickinessInvalidALB (67.80s)
--- PASS: TestAccAWSLBTargetGroup_stickinessInvalidNLB (49.79s)
--- PASS: TestAccAWSLBTargetGroup_stickinessValidALB (59.67s)
--- PASS: TestAccAWSLBTargetGroup_stickinessValidNLB (150.14s)
--- PASS: TestAccAWSLBTargetGroup_tags (129.29s)
--- PASS: TestAccAWSLBTargetGroup_TCP_HTTPHealthCheck (83.40s)
--- PASS: TestAccAWSLBTargetGroup_updateHealthCheck (101.32s)
--- PASS: TestAccAWSLBTargetGroup_updateSticknessEnabled (129.48s)
--- PASS: TestAccAWSLBTargetGroup_withoutHealthcheck (34.78s)
--- SKIP: TestAccAWSLBTargetGroup_Protocol_Geneve (1.82s)
```

* Update CHANGELOG for #18634

.changelog/18634.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+resource/aws_lb_target_group: Handle read-after-create eventual consistency
+```

aws/internal/service/elbv2/finder/finder.go

@@ -0,0 +1,36 @@
+package finder
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/elbv2"
+)
+
+func TargetGroupByARN(conn *elbv2.ELBV2, arn string) (*elbv2.TargetGroup, error) {
+	input := &elbv2.DescribeTargetGroupsInput{
+		TargetGroupArns: aws.StringSlice([]string{arn}),
+	}
+
+	output, err := conn.DescribeTargetGroups(input)
+
+	if err != nil {
+		return nil, err
+	}
+
+	if output == nil {
+		return nil, nil
+	}
+
+	for _, targetGroup := range output.TargetGroups {
+		if targetGroup == nil {
+			continue
+		}
+
+		if aws.StringValue(targetGroup.TargetGroupArn) != arn {
+			continue
+		}
+
+		return targetGroup, nil
+	}
+
+	return nil, nil
+}

aws/internal/service/elbv2/waiter/waiter.go

@@ -29,6 +29,8 @@ const (
 	LoadBalancerListenerCreateTimeout = 5 * time.Minute
 	LoadBalancerListenerReadTimeout   = 2 * time.Minute
 	LoadBalancerListenerUpdateTimeout = 5 * time.Minute
+
+	PropagationTimeout = 2 * time.Minute
 )
 
 // LoadBalancerActive waits for a Load Balancer to return active

aws/resource_aws_lb_target_group.go

@@ -16,6 +16,7 @@ import (
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
 	"github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags"
+	"github.com/terraform-providers/terraform-provider-aws/aws/internal/service/elbv2/finder"
 	"github.com/terraform-providers/terraform-provider-aws/aws/internal/service/elbv2/waiter"
 	"github.com/terraform-providers/terraform-provider-aws/aws/internal/tfresource"
 )
@@ -371,23 +372,55 @@ func resourceAwsLbTargetGroupCreate(d *schema.ResourceData, meta interface{}) er
 func resourceAwsLbTargetGroupRead(d *schema.ResourceData, meta interface{}) error {
 	conn := meta.(*AWSClient).elbv2conn
 
-	resp, err := conn.DescribeTargetGroups(&elbv2.DescribeTargetGroupsInput{
-		TargetGroupArns: aws.StringSlice([]string{d.Id()}),
+	var targetGroup *elbv2.TargetGroup
+
+	err := resource.Retry(waiter.PropagationTimeout, func() *resource.RetryError {
+		var err error
+
+		targetGroup, err = finder.TargetGroupByARN(conn, d.Id())
+
+		if d.IsNewResource() && tfawserr.ErrCodeEquals(err, elbv2.ErrCodeTargetGroupNotFoundException) {
+			return resource.RetryableError(err)
+		}
+
+		if err != nil {
+			return resource.NonRetryableError(err)
+		}
+
+		if d.IsNewResource() && targetGroup == nil {
+			return resource.RetryableError(&resource.NotFoundError{
+				LastError: fmt.Errorf("ELBv2 Target Group (%s) not found", d.Id()),
+			})
+		}
+
+		return nil
 	})
+
+	if tfresource.TimedOut(err) {
+		targetGroup, err = finder.TargetGroupByARN(conn, d.Id())
+	}
+
+	if !d.IsNewResource() && tfawserr.ErrCodeEquals(err, elbv2.ErrCodeTargetGroupNotFoundException) {
+		log.Printf("[WARN] ELBv2 Target Group (%s) not found, removing from state", d.Id())
+		d.SetId("")
+		return nil
+	}
+
 	if err != nil {
-		if isAWSErr(err, elbv2.ErrCodeTargetGroupNotFoundException, "") {
-			log.Printf("[DEBUG] DescribeTargetGroups - removing %s from state", d.Id())
-			d.SetId("")
-			return nil
-		}
-		return fmt.Errorf("error retrieving Target Group: %w", err)
+		return fmt.Errorf("error reading ELBv2 Target Group (%s): %w", d.Id(), err)
 	}
 
-	if len(resp.TargetGroups) != 1 {
-		return fmt.Errorf("error retrieving Target Group %q", d.Id())
+	if targetGroup == nil {
+		if d.IsNewResource() {
+			return fmt.Errorf("error reading ELBv2 Target Group (%s): not found after creation", d.Id())
+		}
+
+		log.Printf("[WARN] ELBv2 Target Group (%s) not found, removing from state", d.Id())
+		d.SetId("")
+		return nil
 	}
 
-	return flattenAwsLbTargetGroupResource(d, meta, resp.TargetGroups[0])
+	return flattenAwsLbTargetGroupResource(d, meta, targetGroup)
 }
 
 func resourceAwsLbTargetGroupUpdate(d *schema.ResourceData, meta interface{}) error {