Skip to content

Commit

Permalink
migrate private endpoints service to use ASO framework
Browse files Browse the repository at this point in the history 
CAPZ sets the following ASO fields if found:
- ApplicationSecurityGroups
- AzureName
- CustomNetworkInterfaceName
- IpConfigurations
- Location
- ManualPrivateLinkServiceConnections
- Owner
- PrivateLinkServiceConnections
- Subnet
- Tags

ASO fields not managed by CAPZ
- ExtendedLocation
  • Loading branch information
@nawazkh
nawazkh committed Dec 1, 2023
1 parent 7c1a8a2 commit 1b4f345
Show file tree
Hide file tree
Showing 19 changed files with 1,772 additions and 1,147 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ WEBHOOK_ROOT ?= $(MANIFEST_ROOT)/webhook
RBAC_ROOT ?= $(MANIFEST_ROOT)/rbac
ASO_CRDS_PATH := $(MANIFEST_ROOT)/aso/crds.yaml
ASO_VERSION := v2.4.0
ASO_CRDS := resourcegroups.resources.azure.com natgateways.network.azure.com managedclusters.containerservice.azure.com managedclustersagentpools.containerservice.azure.com bastionhosts.network.azure.com
ASO_CRDS := resourcegroups.resources.azure.com natgateways.network.azure.com managedclusters.containerservice.azure.com managedclustersagentpools.containerservice.azure.com bastionhosts.network.azure.com privateendpoints.network.azure.com

# Allow overriding the imagePullPolicy
PULL_POLICY ?= Always
Expand Down
73 changes: 32 additions & 41 deletions azure/scope/cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1074,54 +1074,45 @@ func (s *ClusterScope) SetAnnotation(key, value string) {
}

// PrivateEndpointSpecs returns the private endpoint specs.
func (s *ClusterScope) PrivateEndpointSpecs() []azure.ResourceSpecGetter {
numberOfSubnets := len(s.AzureCluster.Spec.NetworkSpec.Subnets)
func (s *ClusterScope) PrivateEndpointSpecs() []azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint] {
subnetsList := s.AzureCluster.Spec.NetworkSpec.Subnets
numberOfSubnets := len(subnetsList)
if s.IsAzureBastionEnabled() {
subnetsList = append(subnetsList, s.AzureCluster.Spec.BastionSpec.AzureBastion.Subnet)

Check warning on line 1081 in azure/scope/cluster.go

View check run for this annotation

Codecov / codecov/patch

azure/scope/cluster.go#L1081 

Added line #L1081 was not covered by tests
numberOfSubnets++
}

privateEndpointSpecs := make([]azure.ResourceSpecGetter, 0, numberOfSubnets)

subnets := s.AzureCluster.Spec.NetworkSpec.Subnets
if s.IsAzureBastionEnabled() {
subnets = append(subnets, s.AzureCluster.Spec.BastionSpec.AzureBastion.Subnet)
}

for _, subnet := range subnets {
privateEndpointSpecs = append(privateEndpointSpecs, s.getPrivateEndpoints(subnet)...)
}

return privateEndpointSpecs
}

func (s *ClusterScope) getPrivateEndpoints(subnet infrav1.SubnetSpec) []azure.ResourceSpecGetter {
privateEndpointSpecs := make([]azure.ResourceSpecGetter, 0)

for _, privateEndpoint := range subnet.PrivateEndpoints {
privateEndpointSpec := &privateendpoints.PrivateEndpointSpec{
Name: privateEndpoint.Name,
ResourceGroup: s.ResourceGroup(),
Location: privateEndpoint.Location,
CustomNetworkInterfaceName: privateEndpoint.CustomNetworkInterfaceName,
PrivateIPAddresses: privateEndpoint.PrivateIPAddresses,
SubnetID: subnet.ID,
ApplicationSecurityGroups: privateEndpoint.ApplicationSecurityGroups,
ManualApproval: privateEndpoint.ManualApproval,
ClusterName: s.ClusterName(),
AdditionalTags: s.AdditionalTags(),
}
// privateEndpointSpecs will be an empty list if no private endpoints were found.
// We pre-allocate the list to avoid unnecessary allocations during append.
privateEndpointSpecs := make([]azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint], 0, numberOfSubnets)

for _, subnet := range subnetsList {
for _, privateEndpoint := range subnet.PrivateEndpoints {
privateEndpointSpec := &privateendpoints.PrivateEndpointSpec{
Name: privateEndpoint.Name,
Namespace: s.Namespace(),
ResourceGroup: s.ResourceGroup(),
Location: privateEndpoint.Location,
CustomNetworkInterfaceName: privateEndpoint.CustomNetworkInterfaceName,
PrivateIPAddresses: privateEndpoint.PrivateIPAddresses,
SubnetID: subnet.ID,
ApplicationSecurityGroups: privateEndpoint.ApplicationSecurityGroups,
ManualApproval: privateEndpoint.ManualApproval,
ClusterName: s.ClusterName(),
AdditionalTags: s.AdditionalTags(),
}

for _, privateLinkServiceConnection := range privateEndpoint.PrivateLinkServiceConnections {
pl := privateendpoints.PrivateLinkServiceConnection{
PrivateLinkServiceID: privateLinkServiceConnection.PrivateLinkServiceID,
Name: privateLinkServiceConnection.Name,
RequestMessage: privateLinkServiceConnection.RequestMessage,
GroupIDs: privateLinkServiceConnection.GroupIDs,
for _, privateLinkServiceConnection := range privateEndpoint.PrivateLinkServiceConnections {
pl := privateendpoints.PrivateLinkServiceConnection{
PrivateLinkServiceID: privateLinkServiceConnection.PrivateLinkServiceID,
Name: privateLinkServiceConnection.Name,
RequestMessage: privateLinkServiceConnection.RequestMessage,
GroupIDs: privateLinkServiceConnection.GroupIDs,
}
privateEndpointSpec.PrivateLinkServiceConnections = append(privateEndpointSpec.PrivateLinkServiceConnections, pl)
}
privateEndpointSpec.PrivateLinkServiceConnections = append(privateEndpointSpec.PrivateLinkServiceConnections, pl)
privateEndpointSpecs = append(privateEndpointSpecs, privateEndpointSpec)
}

privateEndpointSpecs = append(privateEndpointSpecs, privateEndpointSpec)
}

return privateEndpointSpecs
Expand Down
244 changes: 244 additions & 0 deletions azure/scope/cluster_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (
"sigs.k8s.io/cluster-api-provider-azure/azure/services/bastionhosts"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/loadbalancers"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/natgateways"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/privateendpoints"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/publicips"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/routetables"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/securitygroups"
Expand Down Expand Up @@ -3388,6 +3389,249 @@ func TestVNetPeerings(t *testing.T) {
}
}

func TestPrivateEndpointSpecs(t *testing.T) {
tests := []struct {
name string
clusterScope ClusterScope
want []azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint]
}{
{
name: "returns empty private endpoints list if no subnets are specified",
clusterScope: ClusterScope{
AzureCluster: &infrav1.AzureCluster{
Spec: infrav1.AzureClusterSpec{
NetworkSpec: infrav1.NetworkSpec{
Subnets: infrav1.Subnets{},
},
},
},
cache: &ClusterCache{},
},
want: make([]azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint], 0),
},
{
name: "returns empty private endpoints list if no private endpoints are specified",
clusterScope: ClusterScope{
AzureCluster: &infrav1.AzureCluster{
Spec: infrav1.AzureClusterSpec{
NetworkSpec: infrav1.NetworkSpec{
Subnets: []infrav1.SubnetSpec{
{
SubnetClassSpec: infrav1.SubnetClassSpec{
PrivateEndpoints: infrav1.PrivateEndpoints{},
},
},
},
},
},
},
cache: &ClusterCache{},
},
want: make([]azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint], 0),
},
{
name: "returns list of private endpoint specs if private endpoints are specified",
clusterScope: ClusterScope{
Cluster: &clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "my-cluster",
Namespace: "dummy-ns",
},
},
AzureCluster: &infrav1.AzureCluster{
Spec: infrav1.AzureClusterSpec{
ResourceGroup: "dummy-rg",
NetworkSpec: infrav1.NetworkSpec{
Subnets: []infrav1.SubnetSpec{
{
ID: "dummy-subnet-id",
SubnetClassSpec: infrav1.SubnetClassSpec{
PrivateEndpoints: []infrav1.PrivateEndpointSpec{
{
Name: "my-private-endpoint",
Location: "westus2",
CustomNetworkInterfaceName: "my-custom-nic",
PrivateIPAddresses: []string{
"IP1",
"IP2",
},
ApplicationSecurityGroups: []string{
"ASG1",
"ASG2",
},
PrivateLinkServiceConnections: []infrav1.PrivateLinkServiceConnection{
{
Name: "my-pls-connection",
RequestMessage: "my-request-message",
PrivateLinkServiceID: "my-pls-id",
GroupIDs: []string{
"my-group-id-1",
},
},
},
},
{
Name: "my-private-endpoint-2",
Location: "westus2",
CustomNetworkInterfaceName: "my-custom-nic-2",
PrivateIPAddresses: []string{
"IP3",
"IP4",
},
ApplicationSecurityGroups: []string{
"ASG3",
"ASG4",
},
PrivateLinkServiceConnections: []infrav1.PrivateLinkServiceConnection{
{
Name: "my-pls-connection",
RequestMessage: "my-request-message",
PrivateLinkServiceID: "my-pls-id",
GroupIDs: []string{
"my-group-id-1",
},
},
},
},
},
},
},
{
ID: "dummy-subnet-id-2",
SubnetClassSpec: infrav1.SubnetClassSpec{
PrivateEndpoints: []infrav1.PrivateEndpointSpec{
{
Name: "my-private-endpoint-3",
Location: "westus2",
CustomNetworkInterfaceName: "my-custom-nic-3",
PrivateIPAddresses: []string{
"IP5",
"IP6",
},
ApplicationSecurityGroups: []string{
"ASG5",
"ASG6",
},
PrivateLinkServiceConnections: []infrav1.PrivateLinkServiceConnection{
{
Name: "my-pls-connection",
RequestMessage: "my-request-message",
PrivateLinkServiceID: "my-pls-id",
GroupIDs: []string{
"my-group-id-1",
},
},
},
},
},
},
},
},
},
},
},
cache: &ClusterCache{},
},
want: []azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint]{
&privateendpoints.PrivateEndpointSpec{
Name: "my-private-endpoint",
Namespace: "dummy-ns",
ResourceGroup: "dummy-rg",
Location: "westus2",
CustomNetworkInterfaceName: "my-custom-nic",
PrivateIPAddresses: []string{
"IP1",
"IP2",
},
SubnetID: "dummy-subnet-id",
ApplicationSecurityGroups: []string{
"ASG1",
"ASG2",
},
ClusterName: "my-cluster",
PrivateLinkServiceConnections: []privateendpoints.PrivateLinkServiceConnection{
{
Name: "my-pls-connection",
RequestMessage: "my-request-message",
PrivateLinkServiceID: "my-pls-id",
GroupIDs: []string{
"my-group-id-1",
},
},
},
AdditionalTags: make(infrav1.Tags, 0),
},
&privateendpoints.PrivateEndpointSpec{
Name: "my-private-endpoint-2",
Namespace: "dummy-ns",
ResourceGroup: "dummy-rg",
Location: "westus2",
CustomNetworkInterfaceName: "my-custom-nic-2",
PrivateIPAddresses: []string{
"IP3",
"IP4",
},
SubnetID: "dummy-subnet-id",
ApplicationSecurityGroups: []string{
"ASG3",
"ASG4",
},
ClusterName: "my-cluster",
PrivateLinkServiceConnections: []privateendpoints.PrivateLinkServiceConnection{
{
Name: "my-pls-connection",
RequestMessage: "my-request-message",
PrivateLinkServiceID: "my-pls-id",
GroupIDs: []string{
"my-group-id-1",
},
},
},
AdditionalTags: make(infrav1.Tags, 0),
},
&privateendpoints.PrivateEndpointSpec{
Name: "my-private-endpoint-3",
Namespace: "dummy-ns",
ResourceGroup: "dummy-rg",
Location: "westus2",
CustomNetworkInterfaceName: "my-custom-nic-3",
PrivateIPAddresses: []string{
"IP5",
"IP6",
},
SubnetID: "dummy-subnet-id-2",
ApplicationSecurityGroups: []string{
"ASG5",
"ASG6",
},
ClusterName: "my-cluster",
PrivateLinkServiceConnections: []privateendpoints.PrivateLinkServiceConnection{
{
Name: "my-pls-connection",
RequestMessage: "my-request-message",
PrivateLinkServiceID: "my-pls-id",
GroupIDs: []string{
"my-group-id-1",
},
},
},
AdditionalTags: make(infrav1.Tags, 0),
},
},
},
}

for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
if got := tt.clusterScope.PrivateEndpointSpecs(); !reflect.DeepEqual(got, tt.want) {
t.Errorf("PrivateEndpointSpecs() = %s, want %s", specArrayToString(got), specArrayToString(tt.want))
}
})
}
}

func TestSetFailureDomain(t *testing.T) {
t.Parallel()

Expand Down
7 changes: 4 additions & 3 deletions azure/scope/managedcontrolplane.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"time"

asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20230201"
asonetworkv1 "github.com/Azure/azure-service-operator/v2/api/network/v1api20220701"
asoresourcesv1 "github.com/Azure/azure-service-operator/v2/api/resources/v1api20200601"
"github.com/pkg/errors"
"golang.org/x/mod/semver"
Expand Down Expand Up @@ -860,12 +861,13 @@ func (s *ManagedControlPlaneScope) AvailabilityStatusFilter(cond *clusterv1.Cond
}

// PrivateEndpointSpecs returns the private endpoint specs.
func (s *ManagedControlPlaneScope) PrivateEndpointSpecs() []azure.ResourceSpecGetter {
privateEndpointSpecs := make([]azure.ResourceSpecGetter, len(s.ControlPlane.Spec.VirtualNetwork.Subnet.PrivateEndpoints))
func (s *ManagedControlPlaneScope) PrivateEndpointSpecs() []azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint] {
privateEndpointSpecs := make([]azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint], 0, len(s.ControlPlane.Spec.VirtualNetwork.Subnet.PrivateEndpoints))

for _, privateEndpoint := range s.ControlPlane.Spec.VirtualNetwork.Subnet.PrivateEndpoints {
privateEndpointSpec := &privateendpoints.PrivateEndpointSpec{
Name: privateEndpoint.Name,
Namespace: s.Cluster.Namespace,
ResourceGroup: s.VNetSpec().ResourceGroupName(),
Location: privateEndpoint.Location,
CustomNetworkInterfaceName: privateEndpoint.CustomNetworkInterfaceName,
Expand All @@ -891,7 +893,6 @@ func (s *ManagedControlPlaneScope) PrivateEndpointSpecs() []azure.ResourceSpecGe
}
privateEndpointSpec.PrivateLinkServiceConnections = append(privateEndpointSpec.PrivateLinkServiceConnections, pl)
}

privateEndpointSpecs = append(privateEndpointSpecs, privateEndpointSpec)
}

Expand Down
Loading

0 comments on commit 1b4f345

Please sign in to comment.