azuread_group: remediate replication delays when adding/removing group members

azuread/helpers/graph/group.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"time"
 
 	"github.com/Azure/azure-sdk-for-go/services/graphrbac/1.6/graphrbac"
 )
@@ -121,8 +122,22 @@ func GroupAddMember(client graphrbac.GroupsClient, ctx context.Context, groupId
 	}
 
 	log.Printf("[DEBUG] Adding member with id %q to Azure AD group with id %q", member, groupId)
-	if _, err := client.AddMember(ctx, groupId, properties); err != nil {
-		return fmt.Errorf("Error adding group member %q to Azure AD Group with ID %q: %+v", member, groupId, err)
+	var err error
+	attempts := 10
+	for i := 0; i <= attempts; i++ {
+		if _, err = client.AddMember(ctx, groupId, properties); err == nil {
+			break
+		}
+		if i == attempts {
+			return fmt.Errorf("Error adding group member %q to Azure AD Group with ID %q: %+v", member, groupId, err)
+		}
+		time.Sleep(time.Second * 2)
+	}
+
+	if _, err := WaitForListAdd(member, func() ([]string, error) {
+		return GroupAllMembers(client, ctx, groupId)
+	}); err != nil {
+		return fmt.Errorf("Error waiting for group membership: %+v", err)
 	}
 
 	return nil

azuread/helpers/graph/replication.go

@@ -34,3 +34,53 @@ func WaitForCreationReplication(f func() (interface{}, error)) (interface{}, err
 		},
 	}).WaitForState()
 }
+
+func WaitForListAdd(item string, f func() ([]string, error)) (interface{}, error) {
+	return (&resource.StateChangeConf{
+		Pending:                   []string{"404"},
+		Target:                    []string{"Found"},
+		Timeout:                   5 * time.Minute,
+		MinTimeout:                1 * time.Second,
+		ContinuousTargetOccurence: 10,
+		Refresh: func() (interface{}, string, error) {
+			listItems, err := f()
+
+			if err != nil {
+				return listItems, "Error", err
+			}
+
+			for _, v := range listItems {
+				if v == item {
+					return listItems, "Found", nil
+				}
+			}
+
+			return listItems, "404", nil
+		},
+	}).WaitForState()
+}
+
+func WaitForListRemove(item string, f func() ([]string, error)) (interface{}, error) {
+	return (&resource.StateChangeConf{
+		Pending:                   []string{"Found"},
+		Target:                    []string{"404"},
+		Timeout:                   5 * time.Minute,
+		MinTimeout:                1 * time.Second,
+		ContinuousTargetOccurence: 10,
+		Refresh: func() (interface{}, string, error) {
+			listItems, err := f()
+
+			if err != nil {
+				return listItems, "Error", err
+			}
+
+			for _, v := range listItems {
+				if v == item {
+					return listItems, "Found", nil
+				}
+			}
+
+			return listItems, "404", nil
+		},
+	}).WaitForState()
+}

azuread/resource_group.go

@@ -3,8 +3,10 @@ package azuread
 import (
 	"fmt"
 	"log"
+	"time"
 
 	"github.com/Azure/azure-sdk-for-go/services/graphrbac/1.6/graphrbac"
+	"github.com/Azure/go-autorest/autorest"
 	"github.com/google/uuid"
 	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
 	"github.com/hashicorp/terraform-plugin-sdk/helper/validation"
@@ -186,11 +188,26 @@ func resourceGroupUpdate(d *schema.ResourceData, meta interface{}) error {
 		membersToAdd := slices.Difference(desiredMembers, existingMembers)
 
 		for _, existingMember := range membersForRemoval {
+			var err error
+			var resp autorest.Response
 			log.Printf("[DEBUG] Removing member with id %q from Azure AD group with id %q", existingMember, d.Id())
-			if resp, err := client.RemoveMember(ctx, d.Id(), existingMember); err != nil {
-				if !ar.ResponseWasNotFound(resp) {
+			attempts := 10
+			for i := 0; i <= attempts; i++ {
+				if resp, err = client.RemoveMember(ctx, d.Id(), existingMember); err != nil {
+					if ar.ResponseWasNotFound(resp) {
+						break
+					}
+				}
+				if i == attempts {
 					return fmt.Errorf("Error Deleting group member %q from Azure AD Group with ID %q: %+v", existingMember, d.Id(), err)
 				}
+				time.Sleep(time.Second * 2)
+			}
+
+			if _, err := graph.WaitForListRemove(existingMember, func() ([]string, error) {
+				return graph.GroupAllMembers(client, ctx, d.Id())
+			}); err != nil {
+				return fmt.Errorf("Error waiting for group membership removal: %+v", err)
 			}
 		}