Make inbound NAT rules reconcile/delete async

azure/converters/inboundnatrules.go

@@ -0,0 +1,34 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package converters
+
+import (
+	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-02-01/network"
+)
+
+type ExistingInboundNatSpec struct {
+	Name         string
+	FrontendPort int32
+}
+
+// InboundNatRuleToExistingInboundNatSpec converts a network.InboundNatRule into an ExistingInboundNatSpec.
+func InboundNatRuleToExistingInboundNatSpec(rule network.InboundNatRule) ExistingInboundNatSpec {
+	return ExistingInboundNatSpec{
+		Name:         *rule.Name,
+		FrontendPort: *rule.InboundNatRulePropertiesFormat.FrontendPort,
+	}
+}

azure/interfaces.go

@@ -62,6 +62,7 @@ type NetworkDescriber interface {
 	SetSubnet(infrav1.SubnetSpec)
 	IsIPv6Enabled() bool
 	ControlPlaneRouteTable() infrav1.RouteTable
+	APIServerLB() *infrav1.LoadBalancerSpec
 	APIServerLBName() string
 	APIServerLBPoolName(string) string
 	IsAPIServerPrivate() bool

azure/scope/machine.go

@@ -38,8 +38,10 @@ import (
 
 	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
 	"sigs.k8s.io/cluster-api-provider-azure/azure"
+	"sigs.k8s.io/cluster-api-provider-azure/azure/converters"
 	"sigs.k8s.io/cluster-api-provider-azure/azure/services/availabilitysets"
 	"sigs.k8s.io/cluster-api-provider-azure/azure/services/disks"
+	"sigs.k8s.io/cluster-api-provider-azure/azure/services/inboundnatrules"
 	"sigs.k8s.io/cluster-api-provider-azure/azure/services/resourceskus"
 	"sigs.k8s.io/cluster-api-provider-azure/azure/services/virtualmachines"
 	"sigs.k8s.io/cluster-api-provider-azure/util/futures"
@@ -193,16 +195,25 @@ func (m *MachineScope) PublicIPSpecs() []azure.PublicIPSpec {
 }
 
 // InboundNatSpecs returns the inbound NAT specs.
-func (m *MachineScope) InboundNatSpecs() []azure.InboundNatSpec {
+func (m *MachineScope) InboundNatSpecs(existingRules []converters.ExistingInboundNatSpec) []azure.ResourceSpecGetter {
+	// The existing inbound NAT rules are needed in order to find an available SSH port for each new inbound NAT rule.
 	if m.Role() == infrav1.ControlPlane {
-		return []azure.InboundNatSpec{
-			{
-				Name:             m.Name(),
-				LoadBalancerName: m.APIServerLBName(),
-			},
+		spec := &inboundnatrules.InboundNatSpec{
+			Name:                      m.Name(),
+			ResourceGroup:             m.ResourceGroup(),
+			LoadBalancerName:          m.APIServerLBName(),
+			FrontendIPConfigurationID: nil,
+			ExistingRules:             existingRules,
 		}
+		if frontEndIPs := m.APIServerLB().FrontendIPs; len(frontEndIPs) > 0 {
+			ipConfig := frontEndIPs[0].Name
+			id := azure.FrontendIPConfigID(m.SubscriptionID(), m.ResourceGroup(), m.APIServerLBName(), ipConfig)
+			spec.FrontendIPConfigurationID = to.StringPtr(id)
+		}
+
+		return []azure.ResourceSpecGetter{spec}
 	}
-	return []azure.InboundNatSpec{}
+	return []azure.ResourceSpecGetter{}
 }
 
 // NICSpecs returns the network interface specs.

azure/scope/machine_test.go

@@ -18,6 +18,7 @@ package scope
 
 import (
 	"context"
+	"fmt"
 	"reflect"
 	"testing"
 
@@ -31,7 +32,9 @@ import (
 
 	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
 	"sigs.k8s.io/cluster-api-provider-azure/azure"
+	"sigs.k8s.io/cluster-api-provider-azure/azure/converters"
 	"sigs.k8s.io/cluster-api-provider-azure/azure/services/disks"
+	"sigs.k8s.io/cluster-api-provider-azure/azure/services/inboundnatrules"
 )
 
 func TestMachineScope_Name(t *testing.T) {
@@ -283,7 +286,7 @@ func TestMachineScope_InboundNatSpecs(t *testing.T) {
 	tests := []struct {
 		name         string
 		machineScope MachineScope
-		want         []azure.InboundNatSpec
+		want         []azure.ResourceSpecGetter
 	}{
 		{
 			name: "returns empty when infra is not control plane",
@@ -295,7 +298,7 @@ func TestMachineScope_InboundNatSpecs(t *testing.T) {
 					},
 				},
 			},
-			want: []azure.InboundNatSpec{},
+			want: []azure.ResourceSpecGetter{},
 		},
 		{
 			name: "returns InboundNatSpec when infra is control plane",
@@ -313,29 +316,58 @@ func TestMachineScope_InboundNatSpecs(t *testing.T) {
 					},
 				},
 				ClusterScoper: &ClusterScope{
+					AzureClients: AzureClients{
+						EnvironmentSettings: auth.EnvironmentSettings{
+							Values: map[string]string{
+								auth.SubscriptionID: "123",
+							},
+						},
+					},
 					AzureCluster: &infrav1.AzureCluster{
 						Spec: infrav1.AzureClusterSpec{
+							ResourceGroup:  "my-rg",
+							SubscriptionID: "123",
 							NetworkSpec: infrav1.NetworkSpec{
 								APIServerLB: infrav1.LoadBalancerSpec{
 									Name: "foo-loadbalancer",
+									FrontendIPs: []infrav1.FrontendIP{
+										{
+											Name: "foo-frontend-ip",
+										},
+									},
 								},
 							},
 						},
 					},
 				},
 			},
-			want: []azure.InboundNatSpec{
-				{
-					Name:             "machine-name",
-					LoadBalancerName: "foo-loadbalancer",
+			want: []azure.ResourceSpecGetter{
+				&inboundnatrules.InboundNatSpec{
+					Name:                      "machine-name",
+					LoadBalancerName:          "foo-loadbalancer",
+					ResourceGroup:             "my-rg",
+					FrontendIPConfigurationID: to.StringPtr(azure.FrontendIPConfigID("123", "my-rg", "foo-loadbalancer", "foo-frontend-ip")),
+					ExistingRules:             []converters.ExistingInboundNatSpec{},
 				},
 			},
 		},
 	}
 	for _, tt := range tests {
+		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
-			if got := tt.machineScope.InboundNatSpecs(); !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("InboundNatSpecs() = %v, want %v", got, tt.want)
+			t.Parallel()
+			if got := tt.machineScope.InboundNatSpecs([]converters.ExistingInboundNatSpec{}); !reflect.DeepEqual(got, tt.want) {
+				gotArray := "[ "
+				for _, spec := range got {
+					gotArray += fmt.Sprintf("%+v ", spec)
+				}
+				gotArray += "]"
+				wantArray := "[ "
+				for _, spec := range tt.want {
+					wantArray += fmt.Sprintf("%+v ", spec)
+				}
+				wantArray += "]"
+				t.Errorf("InboundNatSpecs([]converters.ExistingInboundNatSpec{}) = %s, want %s", gotArray, wantArray)
 			}
 		})
 	}

azure/scope/managedcontrolplane.go

@@ -316,6 +316,11 @@ func (s *ManagedControlPlaneScope) IsVnetManaged() bool {
 	return true
 }
 
+// APIServerLBName returns the API Server LB spec.
+func (s *ManagedControlPlaneScope) APIServerLB() *infrav1.LoadBalancerSpec {
+	return nil // does not apply for AKS
+}
+
 // APIServerLBName returns the API Server LB name.
 func (s *ManagedControlPlaneScope) APIServerLBName() string {
 	return "" // does not apply for AKS