resource/aws_synthetics_canary: Handle asynchronous IAM eventual cons…

…istency error on creation (#18404)

* resource/aws_synthetics_canary: Handle asynchronous IAM eventual consistency error on creation

Reference: #18101

Previously:

```
=== CONT  TestAccAWSSyntheticsCanary_basic
resource_aws_synthetics_canary_test.go:82: Step 1/3 error: Error running apply: exit status 1
Error: error waiting for Synthetics Canary (tf-acc-test-s9t49btk) creation: : The role defined for the function cannot be assumed by Lambda. (Service: AWSLambda; Status Code: 400; Error Code: InvalidParameterValueException; Request ID: 8e3169db-124d-4c5a-ac0e-d9f382cf737f; Proxy: null)
on terraform_plugin_test.tf line 92, in resource "aws_synthetics_canary" "test":
92: resource "aws_synthetics_canary" "test" {
--- FAIL: TestAccAWSSyntheticsCanary_basic (16.37s)

=== CONT  TestAccAWSSyntheticsCanary_runtimeVersion
resource_aws_synthetics_canary_test.go:156: Step 1/3 error: Error running apply: exit status 1
Error: error waiting for Synthetics Canary (tf-acc-test-nenvizbg) creation: : The role defined for the function cannot be assumed by Lambda. (Service: AWSLambda; Status Code: 400; Error Code: InvalidParameterValueException; Request ID: ffcb7edb-c77e-43f2-8a0a-1183e92f0f8c; Proxy: null)
on terraform_plugin_test.tf line 92, in resource "aws_synthetics_canary" "test":
92: resource "aws_synthetics_canary" "test" {
--- FAIL: TestAccAWSSyntheticsCanary_runtimeVersion (16.85s)

=== CONT  TestAccAWSSyntheticsCanary_startCanary_codeChanges
resource_aws_synthetics_canary_test.go:239: Step 1/3 error: Error running apply: exit status 1
Error: error waiting for Synthetics Canary (tf-acc-test-7c9erovn) creation: : The role defined for the function cannot be assumed by Lambda. (Service: AWSLambda; Status Code: 400; Error Code: InvalidParameterValueException; Request ID: 1c4bacd4-3134-4df3-8e82-2538e155b524; Proxy: null)
on terraform_plugin_test.tf line 92, in resource "aws_synthetics_canary" "test":
92: resource "aws_synthetics_canary" "test" {
--- FAIL: TestAccAWSSyntheticsCanary_startCanary_codeChanges (16.92s)

=== CONT  TestAccAWSSyntheticsCanary_startCanary
resource_aws_synthetics_canary_test.go:190: Step 1/4 error: Error running apply: exit status 1
Error: error waiting for Synthetics Canary (tf-acc-test-gzn0ksyz) creation: : The role defined for the function cannot be assumed by Lambda. (Service: AWSLambda; Status Code: 400; Error Code: InvalidParameterValueException; Request ID: afa63236-ac58-4cec-9c93-67192216f734; Proxy: null)
on terraform_plugin_test.tf line 92, in resource "aws_synthetics_canary" "test":
92: resource "aws_synthetics_canary" "test" {
--- FAIL: TestAccAWSSyntheticsCanary_startCanary (17.91s)

=== CONT  TestAccAWSSyntheticsCanary_s3
resource_aws_synthetics_canary_test.go:281: Step 1/2 error: Error running apply: exit status 1
Error: error waiting for Synthetics Canary (tf-acc-test-rhjz4908) creation: : The role defined for the function cannot be assumed by Lambda. (Service: AWSLambda; Status Code: 400; Error Code: InvalidParameterValueException; Request ID: ea96e8db-b243-499e-b6e9-54e5d30c7604; Proxy: null)
on terraform_plugin_test.tf line 92, in resource "aws_synthetics_canary" "test":
92: resource "aws_synthetics_canary" "test" {
--- FAIL: TestAccAWSSyntheticsCanary_s3 (16.68s)

=== CONT  TestAccAWSSyntheticsCanary_runConfig
resource_aws_synthetics_canary_test.go:326: Step 1/4 error: Error running apply: exit status 1
Error: error waiting for Synthetics Canary (tf-acc-test-qy4r4rep) creation: : The role defined for the function cannot be assumed by Lambda. (Service: AWSLambda; Status Code: 400; Error Code: InvalidParameterValueException; Request ID: aee1cc0e-6f2d-4f8d-8720-91aa25333dcc; Proxy: null)
on terraform_plugin_test.tf line 92, in resource "aws_synthetics_canary" "test":
92: resource "aws_synthetics_canary" "test" {
--- FAIL: TestAccAWSSyntheticsCanary_runConfig (15.30s)

=== CONT  TestAccAWSSyntheticsCanary_runConfigTracing
resource_aws_synthetics_canary_test.go:373: Step 1/4 error: Error running apply: exit status 1
Error: error waiting for Synthetics Canary (tf-acc-test-44kmj0af) creation: : The role defined for the function cannot be assumed by Lambda. (Service: AWSLambda; Status Code: 400; Error Code: InvalidParameterValueException; Request ID: 653d7e28-e3a6-4d5d-8da0-818cb7debb9a; Proxy: null)
on terraform_plugin_test.tf line 92, in resource "aws_synthetics_canary" "test":
92: resource "aws_synthetics_canary" "test" {
--- FAIL: TestAccAWSSyntheticsCanary_runConfigTracing (14.80s)

=== CONT  TestAccAWSSyntheticsCanary_vpc
resource_aws_synthetics_canary_test.go:416: Step 1/4 error: Error running apply: exit status 1
Error: error waiting for Synthetics Canary (tf-acc-test-278kyhh1) creation: : The role defined for the function cannot be assumed by Lambda. (Service: AWSLambda; Status Code: 400; Error Code: InvalidParameterValueException; Request ID: 9e377590-2ea9-42db-8340-f181964fd388; Proxy: null)
on terraform_plugin_test.tf line 150, in resource "aws_synthetics_canary" "test":
150: resource "aws_synthetics_canary" "test" {
--- FAIL: TestAccAWSSyntheticsCanary_vpc (15.25s)

=== CONT  TestAccAWSSyntheticsCanary_tags
resource_aws_synthetics_canary_test.go:466: Step 1/4 error: Error running apply: exit status 1
Error: error waiting for Synthetics Canary (tf-acc-test-8thffpcs) creation: : The role defined for the function cannot be assumed by Lambda. (Service: AWSLambda; Status Code: 400; Error Code: InvalidParameterValueException; Request ID: 60d2ea31-fdcb-4c4e-85ec-eecda075d58c; Proxy: null)
on terraform_plugin_test.tf line 92, in resource "aws_synthetics_canary" "test":
92: resource "aws_synthetics_canary" "test" {
--- FAIL: TestAccAWSSyntheticsCanary_tags (15.32s)

=== CONT  TestAccAWSSyntheticsCanary_disappears
resource_aws_synthetics_canary_test.go:513: Step 1/1 error: Error running apply: exit status 1
Error: error waiting for Synthetics Canary (tf-acc-test-yelzrxeu) creation: : The role defined for the function cannot be assumed by Lambda. (Service: AWSLambda; Status Code: 400; Error Code: InvalidParameterValueException; Request ID: 398f9321-06ea-4019-ba08-37594ec32f04; Proxy: null)
on terraform_plugin_test.tf line 92, in resource "aws_synthetics_canary" "test":
92: resource "aws_synthetics_canary" "test" {
--- FAIL: TestAccAWSSyntheticsCanary_disappears (15.34s)
```

Output from acceptance testing:

```
--- PASS: TestAccAWSSyntheticsCanary_basic (105.78s)
--- PASS: TestAccAWSSyntheticsCanary_disappears (59.55s)
--- PASS: TestAccAWSSyntheticsCanary_runConfig (114.95s)
--- PASS: TestAccAWSSyntheticsCanary_runConfigTracing (94.08s)
--- PASS: TestAccAWSSyntheticsCanary_runtimeVersion (89.12s)
--- PASS: TestAccAWSSyntheticsCanary_s3 (72.90s)
--- PASS: TestAccAWSSyntheticsCanary_startCanary (124.81s)
--- PASS: TestAccAWSSyntheticsCanary_startCanary_codeChanges (111.66s)
--- PASS: TestAccAWSSyntheticsCanary_tags (115.10s)
--- PASS: TestAccAWSSyntheticsCanary_vpc (678.91s)
```

* Update CHANGELOG for #18404

.changelog/18404.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+resource/aws_synthetics_canary: Handle asynchronous IAM eventual consistency error on creation
+```

aws/resource_aws_synthetics_canary.go

@@ -10,11 +10,14 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/arn"
 	"github.com/aws/aws-sdk-go/service/synthetics"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
 	"github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags"
+	iamwaiter "github.com/terraform-providers/terraform-provider-aws/aws/internal/service/iam/waiter"
 	"github.com/terraform-providers/terraform-provider-aws/aws/internal/service/synthetics/finder"
 	"github.com/terraform-providers/terraform-provider-aws/aws/internal/service/synthetics/waiter"
+	"github.com/terraform-providers/terraform-provider-aws/aws/internal/tfresource"
 )
 
 const awsMutexCanary = `aws_synthetics_canary`
@@ -259,15 +262,59 @@ func resourceAwsSyntheticsCanaryCreate(d *schema.ResourceData, meta interface{})
 
 	log.Printf("[DEBUG] creating Synthetics Canary: %#v", input)
 
-	resp, err := conn.CreateCanary(input)
-	if err != nil {
-		return fmt.Errorf("error creating Synthetics Canary: %w", err)
-	}
+	// Underlying IAM eventual consistency errors can occur after the creation
+	// operation. The goal is only retry these types of errors up to the IAM
+	// timeout. Since the creation process is asynchronous and can take up to
+	// its own timeout, we store a stop time upfront for checking.
+	iamwaiterStopTime := time.Now().Add(iamwaiter.PropagationTimeout)
+
+	// Ensure to add IAM eventual consistency timeout in case of retries
+	err = resource.Retry(iamwaiter.PropagationTimeout+waiter.CanaryCreatedTimeout, func() *resource.RetryError {
+		// Only retry IAM eventual consistency errors up to that timeout
+		iamwaiterRetry := time.Now().Before(iamwaiterStopTime)
+
+		resp, err := conn.CreateCanary(input)
+
+		if err != nil {
+			return resource.NonRetryableError(fmt.Errorf("error creating Synthetics Canary: %w", err))
+		}
 
-	d.SetId(aws.StringValue(resp.Canary.Name))
+		if resp == nil || resp.Canary == nil {
+			return resource.NonRetryableError(fmt.Errorf("error creating Synthetics Canary: empty response"))
+		}
+
+		d.SetId(aws.StringValue(resp.Canary.Name))
+
+		_, err = waiter.CanaryReady(conn, d.Id())
+
+		if err != nil {
+			// This error synthesized from the Status object and not an AWS SDK Go error type
+			if iamwaiterRetry && strings.Contains(err.Error(), "The role defined for the function cannot be assumed by Lambda") {
+				return resource.RetryableError(fmt.Errorf("error waiting for Synthetics Canary (%s) creation: %w", d.Id(), err))
+			}
 
-	if _, err := waiter.CanaryReady(conn, d.Id()); err != nil {
-		return fmt.Errorf("error waiting for Synthetics Canary (%s) creation: %w", d.Id(), err)
+			return resource.NonRetryableError(fmt.Errorf("error waiting for Synthetics Canary (%s) creation: %w", d.Id(), err))
+		}
+
+		return nil
+	})
+
+	if tfresource.TimedOut(err) {
+		resp, err := conn.CreateCanary(input)
+
+		if err != nil {
+			return fmt.Errorf("error creating Synthetics Canary: %w", err)
+		}
+
+		if resp == nil || resp.Canary == nil {
+			return fmt.Errorf("error creating Synthetics Canary: empty response")
+		}
+
+		d.SetId(aws.StringValue(resp.Canary.Name))
+
+		if _, err = waiter.CanaryReady(conn, d.Id()); err != nil {
+			return fmt.Errorf("error waiting for Synthetics Canary (%s) creation: %w", d.Id(), err)
+		}
 	}
 
 	if v := d.Get("start_canary"); v.(bool) {