Make managedcluster reconcile and delete async

azure/interfaces.go

@@ -106,6 +106,8 @@ type ResourceSpecGetter interface {
 	OwnerResourceName() string
 	// ResourceGroupName returns the name of the resource group the resource is in.
 	ResourceGroupName() string
+	// CustomHeaders returns the headers that should be added to Azure API calls.
+	CustomHeaders() map[string]string
 	// Parameters takes the existing resource and returns the desired parameters of the resource.
 	// If the resource does not exist, or we do not care about existing parameters to update the resource, existing should be nil.
 	// If no update is needed on the resource, Parameters should return nil.

azure/scope/managedcontrolplane.go

@@ -33,10 +33,12 @@ import (
 	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
 	"sigs.k8s.io/cluster-api-provider-azure/azure"
 	"sigs.k8s.io/cluster-api-provider-azure/azure/services/groups"
+	"sigs.k8s.io/cluster-api-provider-azure/azure/services/managedclusters"
 	"sigs.k8s.io/cluster-api-provider-azure/azure/services/subnets"
 	"sigs.k8s.io/cluster-api-provider-azure/azure/services/virtualnetworks"
 	infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1"
 	"sigs.k8s.io/cluster-api-provider-azure/util/futures"
+	"sigs.k8s.io/cluster-api-provider-azure/util/maps"
 	"sigs.k8s.io/cluster-api-provider-azure/util/tele"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
@@ -396,18 +398,26 @@ func (s *ManagedControlPlaneScope) ManagedClusterAnnotations() map[string]string
 }
 
 // ManagedClusterSpec returns the managed cluster spec.
-func (s *ManagedControlPlaneScope) ManagedClusterSpec() (azure.ManagedClusterSpec, error) {
+func (s *ManagedControlPlaneScope) ManagedClusterSpec(ctx context.Context) (azure.ResourceSpecGetter, error) {
 	decodedSSHPublicKey, err := base64.StdEncoding.DecodeString(s.ControlPlane.Spec.SSHPublicKey)
 	if err != nil {
-		return azure.ManagedClusterSpec{}, errors.Wrap(err, "failed to decode SSHPublicKey")
+		return &managedclusters.ManagedClusterSpec{}, errors.Wrap(err, "failed to decode SSHPublicKey")
 	}
 
-	managedClusterSpec := azure.ManagedClusterSpec{
+	// TODO: Are there any side effects if we do this on every reconcile even when it's an update?
+	agentPools, err := s.GetAllAgentPoolSpecs(ctx)
+	if err != nil {
+		return &managedclusters.ManagedClusterSpec{}, errors.Wrapf(err, "failed to get system agent pool specs for managed cluster %s", s.ClusterName())
+	}
+
+	managedClusterSpec := managedclusters.ManagedClusterSpec{
 		Name:                  s.ControlPlane.Name,
-		ResourceGroupName:     s.ControlPlane.Spec.ResourceGroupName,
+		ResourceGroup:         s.ControlPlane.Spec.ResourceGroupName,
 		NodeResourceGroupName: s.ControlPlane.Spec.NodeResourceGroupName,
+		AgentPools:            agentPools,
 		Location:              s.ControlPlane.Spec.Location,
 		Tags:                  s.ControlPlane.Spec.AdditionalTags,
+		Headers:               maps.FilterByKeyPrefix(s.ManagedClusterAnnotations(), azure.CustomHeaderPrefix),
 		Version:               strings.TrimPrefix(s.ControlPlane.Spec.Version, "v"),
 		SSHPublicKey:          string(decodedSSHPublicKey),
 		DNSServiceIP:          s.ControlPlane.Spec.DNSServiceIP,
@@ -434,7 +444,7 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec() (azure.ManagedClusterSpe
 			// A user may provide zero or one CIDR blocks. If they provide an empty array,
 			// we ignore it and use the default. AKS doesn't support > 1 Service/Pod CIDR.
 			if len(clusterNetwork.Services.CIDRBlocks) > 1 {
-				return azure.ManagedClusterSpec{}, errors.New("managed control planes only allow one service cidr")
+				return &managedclusters.ManagedClusterSpec{}, errors.New("managed control planes only allow one service cidr")
 			}
 			if len(clusterNetwork.Services.CIDRBlocks) == 1 {
 				managedClusterSpec.ServiceCIDR = clusterNetwork.Services.CIDRBlocks[0]
@@ -444,7 +454,7 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec() (azure.ManagedClusterSpe
 			// A user may provide zero or one CIDR blocks. If they provide an empty array,
 			// we ignore it and use the default. AKS doesn't support > 1 Service/Pod CIDR.
 			if len(clusterNetwork.Pods.CIDRBlocks) > 1 {
-				return azure.ManagedClusterSpec{}, errors.New("managed control planes only allow one service cidr")
+				return &managedclusters.ManagedClusterSpec{}, errors.New("managed control planes only allow one service cidr")
 			}
 			if len(clusterNetwork.Pods.CIDRBlocks) == 1 {
 				managedClusterSpec.PodCIDR = clusterNetwork.Pods.CIDRBlocks[0]
@@ -454,20 +464,20 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec() (azure.ManagedClusterSpe
 
 	if s.ControlPlane.Spec.DNSServiceIP != nil {
 		if managedClusterSpec.ServiceCIDR == "" {
-			return azure.ManagedClusterSpec{}, fmt.Errorf(s.Cluster.Name + " cluster serviceCIDR must be specified if specifying DNSServiceIP")
+			return &managedclusters.ManagedClusterSpec{}, fmt.Errorf(s.Cluster.Name + " cluster serviceCIDR must be specified if specifying DNSServiceIP")
 		}
 		_, cidr, err := net.ParseCIDR(managedClusterSpec.ServiceCIDR)
 		if err != nil {
-			return azure.ManagedClusterSpec{}, fmt.Errorf("failed to parse cluster service cidr: %w", err)
+			return &managedclusters.ManagedClusterSpec{}, fmt.Errorf("failed to parse cluster service cidr: %w", err)
 		}
 		ip := net.ParseIP(*s.ControlPlane.Spec.DNSServiceIP)
 		if !cidr.Contains(ip) {
-			return azure.ManagedClusterSpec{}, fmt.Errorf(s.ControlPlane.Name + " DNSServiceIP must reside within the associated cluster serviceCIDR")
+			return &managedclusters.ManagedClusterSpec{}, fmt.Errorf(s.ControlPlane.Name + " DNSServiceIP must reside within the associated cluster serviceCIDR")
 		}
 	}
 
 	if s.ControlPlane.Spec.AADProfile != nil {
-		managedClusterSpec.AADProfile = &azure.AADProfile{
+		managedClusterSpec.AADProfile = &managedclusters.AADProfile{
 			Managed:             s.ControlPlane.Spec.AADProfile.Managed,
 			EnableAzureRBAC:     s.ControlPlane.Spec.AADProfile.Managed,
 			AdminGroupObjectIDs: s.ControlPlane.Spec.AADProfile.AdminGroupObjectIDs,
@@ -476,7 +486,7 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec() (azure.ManagedClusterSpe
 
 	if s.ControlPlane.Spec.AddonProfiles != nil {
 		for _, profile := range s.ControlPlane.Spec.AddonProfiles {
-			managedClusterSpec.AddonProfiles = append(managedClusterSpec.AddonProfiles, azure.AddonProfile{
+			managedClusterSpec.AddonProfiles = append(managedClusterSpec.AddonProfiles, managedclusters.AddonProfile{
 				Name:    profile.Name,
 				Enabled: profile.Enabled,
 				Config:  profile.Config,
@@ -485,13 +495,13 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec() (azure.ManagedClusterSpe
 	}
 
 	if s.ControlPlane.Spec.SKU != nil {
-		managedClusterSpec.SKU = &azure.SKU{
+		managedClusterSpec.SKU = &managedclusters.SKU{
 			Tier: string(s.ControlPlane.Spec.SKU.Tier),
 		}
 	}
 
 	if s.ControlPlane.Spec.LoadBalancerProfile != nil {
-		managedClusterSpec.LoadBalancerProfile = &azure.LoadBalancerProfile{
+		managedClusterSpec.LoadBalancerProfile = &managedclusters.LoadBalancerProfile{
 			ManagedOutboundIPs:     s.ControlPlane.Spec.LoadBalancerProfile.ManagedOutboundIPs,
 			OutboundIPPrefixes:     s.ControlPlane.Spec.LoadBalancerProfile.OutboundIPPrefixes,
 			OutboundIPs:            s.ControlPlane.Spec.LoadBalancerProfile.OutboundIPs,
@@ -501,15 +511,15 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec() (azure.ManagedClusterSpe
 	}
 
 	if s.ControlPlane.Spec.APIServerAccessProfile != nil {
-		managedClusterSpec.APIServerAccessProfile = &azure.APIServerAccessProfile{
+		managedClusterSpec.APIServerAccessProfile = &managedclusters.APIServerAccessProfile{
 			AuthorizedIPRanges:             s.ControlPlane.Spec.APIServerAccessProfile.AuthorizedIPRanges,
 			EnablePrivateCluster:           s.ControlPlane.Spec.APIServerAccessProfile.EnablePrivateCluster,
 			PrivateDNSZone:                 s.ControlPlane.Spec.APIServerAccessProfile.PrivateDNSZone,
 			EnablePrivateClusterPublicFQDN: s.ControlPlane.Spec.APIServerAccessProfile.EnablePrivateClusterPublicFQDN,
 		}
 	}
 
-	return managedClusterSpec, nil
+	return &managedClusterSpec, nil
 }
 
 // GetAllAgentPoolSpecs gets a slice of azure.AgentPoolSpec for the list of agent pools.

azure/scope/managedcontrolplane_test.go

@@ -27,6 +27,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"sigs.k8s.io/cluster-api-provider-azure/azure"
+	"sigs.k8s.io/cluster-api-provider-azure/azure/services/managedclusters"
 	infrav1 "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	capiv1exp "sigs.k8s.io/cluster-api/exp/api/v1beta1"
@@ -678,7 +679,7 @@ func TestManagedControlPlaneScope_AddonProfiles(t *testing.T) {
 	cases := []struct {
 		Name     string
 		Input    ManagedControlPlaneScopeParams
-		Expected azure.ManagedClusterSpec
+		Expected managedclusters.ManagedClusterSpec
 	}{
 		{
 			Name: "Without add-ons",
@@ -705,7 +706,7 @@ func TestManagedControlPlaneScope_AddonProfiles(t *testing.T) {
 				InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem),
 				PatchTarget:      getAzureMachinePool("pool0", infrav1.NodePoolModeSystem),
 			},
-			Expected: azure.ManagedClusterSpec{
+			Expected: managedclusters.ManagedClusterSpec{
 				Name:         "cluster1",
 				VnetSubnetID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups//providers/Microsoft.Network/virtualNetworks//subnets/",
 			},
@@ -739,7 +740,7 @@ func TestManagedControlPlaneScope_AddonProfiles(t *testing.T) {
 				InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem),
 				PatchTarget:      getAzureMachinePool("pool0", infrav1.NodePoolModeSystem),
 			},
-			Expected: azure.ManagedClusterSpec{
+			Expected: managedclusters.ManagedClusterSpec{
 				Name:         "cluster1",
 				VnetSubnetID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups//providers/Microsoft.Network/virtualNetworks//subnets/",
 				AddonProfiles: []azure.AddonProfile{

azure/services/availabilitysets/spec.go

@@ -52,6 +52,11 @@ func (s *AvailabilitySetSpec) OwnerResourceName() string {
 	return ""
 }
 
+// CustomHeaders is not implemented for availability sets.
+func (s *AvailabilitySetSpec) CustomHeaders() map[string]string {
+	return nil
+}
+
 // Parameters returns the parameters for the availability set.
 func (s *AvailabilitySetSpec) Parameters(existing interface{}) (params interface{}, err error) {
 	if existing != nil {

azure/services/bastionhosts/azurebastion_spec.go → azure/services/bastionhosts/spec.go

@@ -59,6 +59,11 @@ func (s *AzureBastionSpec) OwnerResourceName() string {
 	return ""
 }
 
+// CustomHeaders is not implemented for bastion hosts.
+func (s *AzureBastionSpec) CustomHeaders() map[string]string {
+	return nil
+}
+
 // Parameters returns the parameters for the bastion host.
 func (s *AzureBastionSpec) Parameters(existing interface{}) (paramteres interface{}, err error) {
 	if existing != nil {

azure/services/disks/spec.go

@@ -37,6 +37,11 @@ func (s *DiskSpec) OwnerResourceName() string {
 	return ""
 }
 
+// CustomHeaders is not implemented for disks.
+func (s *DiskSpec) CustomHeaders() map[string]string {
+	return nil
+}
+
 // Parameters is a no-op for disks.
 func (s *DiskSpec) Parameters(existing interface{}) (params interface{}, err error) {
 	return nil, nil

azure/services/groups/spec.go

@@ -47,6 +47,11 @@ func (s *GroupSpec) OwnerResourceName() string {
 	return "" // not applicable
 }
 
+// CustomHeaders is not implemented for groups.
+func (s *GroupSpec) CustomHeaders() map[string]string {
+	return nil
+}
+
 // Parameters returns the parameters for the group.
 func (s *GroupSpec) Parameters(existing interface{}) (params interface{}, err error) {
 	if existing != nil {

azure/services/inboundnatrules/spec.go

@@ -46,6 +46,11 @@ func (s *InboundNatSpec) OwnerResourceName() string {
 	return s.LoadBalancerName
 }
 
+// CustomHeaders is not implemented for inbound NAT rules.
+func (s *InboundNatSpec) CustomHeaders() map[string]string {
+	return nil
+}
+
 // Parameters returns the parameters for the inbound NAT rule.
 func (s *InboundNatSpec) Parameters(existing interface{}) (parameters interface{}, err error) {
 	if existing != nil {

azure/services/loadbalancers/spec.go

@@ -60,6 +60,11 @@ func (s *LBSpec) OwnerResourceName() string {
 	return ""
 }
 
+// CustomHeaders is not implemented for load balancers.
+func (s *LBSpec) CustomHeaders() map[string]string {
+	return nil
+}
+
 // Parameters returns the parameters for the load balancer.
 func (s *LBSpec) Parameters(existing interface{}) (parameters interface{}, err error) {
 	var (