Skip to content

Commit

Permalink
migrate private endpoints service to use ASO framework
Browse files Browse the repository at this point in the history 
CAPZ sets the following fields on ASO if found:
- ApplicationSecurityGroups
- AzureName
- CustomNetworkInterfaceName
- IpConfigurations
- Location
- ManualPrivateLinkServiceConnections
- Owner
- PrivateLinkServiceConnections
- Subnet
- Tags

Fields not managed by CAPZ
- ExtendedLocation

Changes:
- added private endpoints CRDs to CAPZ
- refactored privateEndpoints spec to ASO
- updated CRDs
- updated role.yaml
- updated spect_test.go
- dropped the redundant unit test of privateendpoint_test.go
  • Loading branch information
@nawazkh
nawazkh committed Oct 25, 2023
1 parent 6e074c7 commit 7c8f0d3
Show file tree
Hide file tree
Showing 19 changed files with 1,740 additions and 1,075 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ WEBHOOK_ROOT ?= $(MANIFEST_ROOT)/webhook
RBAC_ROOT ?= $(MANIFEST_ROOT)/rbac
ASO_CRDS_PATH := $(MANIFEST_ROOT)/aso/crds.yaml
ASO_VERSION := v2.3.0
ASO_CRDS := resourcegroups.resources.azure.com natgateways.network.azure.com
ASO_CRDS := resourcegroups.resources.azure.com privateendpoints.network.azure.com natgateways.network.azure.com

# Allow overriding the imagePullPolicy
PULL_POLICY ?= Always
Expand Down
73 changes: 32 additions & 41 deletions azure/scope/cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1056,54 +1056,45 @@ func (s *ClusterScope) SetAnnotation(key, value string) {
}

// PrivateEndpointSpecs returns the private endpoint specs.
func (s *ClusterScope) PrivateEndpointSpecs() []azure.ResourceSpecGetter {
numberOfSubnets := len(s.AzureCluster.Spec.NetworkSpec.Subnets)
func (s *ClusterScope) PrivateEndpointSpecs() []azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint] {
subnetsList := s.AzureCluster.Spec.NetworkSpec.Subnets
numberOfSubnets := len(subnetsList)
if s.IsAzureBastionEnabled() {
subnetsList = append(subnetsList, s.AzureCluster.Spec.BastionSpec.AzureBastion.Subnet)

Check warning on line 1063 in azure/scope/cluster.go

View check run for this annotation

Codecov / codecov/patch

azure/scope/cluster.go#L1063 

Added line #L1063 was not covered by tests
numberOfSubnets++
}

privateEndpointSpecs := make([]azure.ResourceSpecGetter, 0, numberOfSubnets)

subnets := s.AzureCluster.Spec.NetworkSpec.Subnets
if s.IsAzureBastionEnabled() {
subnets = append(subnets, s.AzureCluster.Spec.BastionSpec.AzureBastion.Subnet)
}

for _, subnet := range subnets {
privateEndpointSpecs = append(privateEndpointSpecs, s.getPrivateEndpoints(subnet)...)
}

return privateEndpointSpecs
}

func (s *ClusterScope) getPrivateEndpoints(subnet infrav1.SubnetSpec) []azure.ResourceSpecGetter {
privateEndpointSpecs := make([]azure.ResourceSpecGetter, 0)

for _, privateEndpoint := range subnet.PrivateEndpoints {
privateEndpointSpec := &privateendpoints.PrivateEndpointSpec{
Name: privateEndpoint.Name,
ResourceGroup: s.ResourceGroup(),
Location: privateEndpoint.Location,
CustomNetworkInterfaceName: privateEndpoint.CustomNetworkInterfaceName,
PrivateIPAddresses: privateEndpoint.PrivateIPAddresses,
SubnetID: subnet.ID,
ApplicationSecurityGroups: privateEndpoint.ApplicationSecurityGroups,
ManualApproval: privateEndpoint.ManualApproval,
ClusterName: s.ClusterName(),
AdditionalTags: s.AdditionalTags(),
}
// privateEndpointSpecs will be an empty list if no private endpoints were found.
// We pre-allocate the list to avoid unnecessary allocations during append.
privateEndpointSpecs := make([]azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint], 0, numberOfSubnets)

for _, subnet := range subnetsList {
for _, privateEndpoint := range subnet.PrivateEndpoints {
privateEndpointSpec := &privateendpoints.PrivateEndpointSpec{
Name: privateEndpoint.Name,
Namespace: s.Namespace(),
ResourceGroup: s.ResourceGroup(),
Location: privateEndpoint.Location,
CustomNetworkInterfaceName: privateEndpoint.CustomNetworkInterfaceName,
PrivateIPAddresses: privateEndpoint.PrivateIPAddresses,
SubnetID: subnet.ID,
ApplicationSecurityGroups: privateEndpoint.ApplicationSecurityGroups,
ManualApproval: privateEndpoint.ManualApproval,
ClusterName: s.ClusterName(),
AdditionalTags: s.AdditionalTags(),
}

for _, privateLinkServiceConnection := range privateEndpoint.PrivateLinkServiceConnections {
pl := privateendpoints.PrivateLinkServiceConnection{
PrivateLinkServiceID: privateLinkServiceConnection.PrivateLinkServiceID,
Name: privateLinkServiceConnection.Name,
RequestMessage: privateLinkServiceConnection.RequestMessage,
GroupIDs: privateLinkServiceConnection.GroupIDs,
for _, privateLinkServiceConnection := range privateEndpoint.PrivateLinkServiceConnections {
pl := privateendpoints.PrivateLinkServiceConnection{
PrivateLinkServiceID: privateLinkServiceConnection.PrivateLinkServiceID,
Name: privateLinkServiceConnection.Name,
RequestMessage: privateLinkServiceConnection.RequestMessage,
GroupIDs: privateLinkServiceConnection.GroupIDs,
}
privateEndpointSpec.PrivateLinkServiceConnections = append(privateEndpointSpec.PrivateLinkServiceConnections, pl)
}
privateEndpointSpec.PrivateLinkServiceConnections = append(privateEndpointSpec.PrivateLinkServiceConnections, pl)
privateEndpointSpecs = append(privateEndpointSpecs, privateEndpointSpec)
}

privateEndpointSpecs = append(privateEndpointSpecs, privateEndpointSpec)
}

return privateEndpointSpecs
Expand Down
244 changes: 244 additions & 0 deletions azure/scope/cluster_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import (
"sigs.k8s.io/cluster-api-provider-azure/azure/services/bastionhosts"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/loadbalancers"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/natgateways"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/privateendpoints"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/publicips"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/routetables"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/securitygroups"
Expand Down Expand Up @@ -3438,3 +3439,246 @@ func TestVNetPeerings(t *testing.T) {
})
}
}

func TestPrivateEndpointSpecs(t *testing.T) {
tests := []struct {
name string
clusterScope ClusterScope
want []azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint]
}{
{
name: "returns empty private endpoints list if no subnets are specified",
clusterScope: ClusterScope{
AzureCluster: &infrav1.AzureCluster{
Spec: infrav1.AzureClusterSpec{
NetworkSpec: infrav1.NetworkSpec{
Subnets: infrav1.Subnets{},
},
},
},
cache: &ClusterCache{},
},
want: make([]azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint], 0),
},
{
name: "returns empty private endpoints list if no private endpoints are specified",
clusterScope: ClusterScope{
AzureCluster: &infrav1.AzureCluster{
Spec: infrav1.AzureClusterSpec{
NetworkSpec: infrav1.NetworkSpec{
Subnets: []infrav1.SubnetSpec{
{
SubnetClassSpec: infrav1.SubnetClassSpec{
PrivateEndpoints: infrav1.PrivateEndpoints{},
},
},
},
},
},
},
cache: &ClusterCache{},
},
want: make([]azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint], 0),
},
{
name: "returns list of private endpoint specs if private endpoints are specified",
clusterScope: ClusterScope{
Cluster: &clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "my-cluster",
Namespace: "dummy-ns",
},
},
AzureCluster: &infrav1.AzureCluster{
Spec: infrav1.AzureClusterSpec{
ResourceGroup: "dummy-rg",
NetworkSpec: infrav1.NetworkSpec{
Subnets: []infrav1.SubnetSpec{
{
ID: "dummy-subnet-id",
SubnetClassSpec: infrav1.SubnetClassSpec{
PrivateEndpoints: []infrav1.PrivateEndpointSpec{
{
Name: "my-private-endpoint",
Location: "westus2",
CustomNetworkInterfaceName: "my-custom-nic",
PrivateIPAddresses: []string{
"IP1",
"IP2",
},
ApplicationSecurityGroups: []string{
"ASG1",
"ASG2",
},
PrivateLinkServiceConnections: []infrav1.PrivateLinkServiceConnection{
{
Name: "my-pls-connection",
RequestMessage: "my-request-message",
PrivateLinkServiceID: "my-pls-id",
GroupIDs: []string{
"my-group-id-1",
},
},
},
},
{
Name: "my-private-endpoint-2",
Location: "westus2",
CustomNetworkInterfaceName: "my-custom-nic-2",
PrivateIPAddresses: []string{
"IP3",
"IP4",
},
ApplicationSecurityGroups: []string{
"ASG3",
"ASG4",
},
PrivateLinkServiceConnections: []infrav1.PrivateLinkServiceConnection{
{
Name: "my-pls-connection",
RequestMessage: "my-request-message",
PrivateLinkServiceID: "my-pls-id",
GroupIDs: []string{
"my-group-id-1",
},
},
},
},
},
},
},
{
ID: "dummy-subnet-id-2",
SubnetClassSpec: infrav1.SubnetClassSpec{
PrivateEndpoints: []infrav1.PrivateEndpointSpec{
{
Name: "my-private-endpoint-3",
Location: "westus2",
CustomNetworkInterfaceName: "my-custom-nic-3",
PrivateIPAddresses: []string{
"IP5",
"IP6",
},
ApplicationSecurityGroups: []string{
"ASG5",
"ASG6",
},
PrivateLinkServiceConnections: []infrav1.PrivateLinkServiceConnection{
{
Name: "my-pls-connection",
RequestMessage: "my-request-message",
PrivateLinkServiceID: "my-pls-id",
GroupIDs: []string{
"my-group-id-1",
},
},
},
},
},
},
},
},
},
},
},
cache: &ClusterCache{},
},
want: []azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint]{
&privateendpoints.PrivateEndpointSpec{
Name: "my-private-endpoint",
Namespace: "dummy-ns",
ResourceGroup: "dummy-rg",
Location: "westus2",
CustomNetworkInterfaceName: "my-custom-nic",
PrivateIPAddresses: []string{
"IP1",
"IP2",
},
SubnetID: "dummy-subnet-id",
ApplicationSecurityGroups: []string{
"ASG1",
"ASG2",
},
ClusterName: "my-cluster",
PrivateLinkServiceConnections: []privateendpoints.PrivateLinkServiceConnection{
{
Name: "my-pls-connection",
RequestMessage: "my-request-message",
PrivateLinkServiceID: "my-pls-id",
GroupIDs: []string{
"my-group-id-1",
},
},
},
AdditionalTags: make(infrav1.Tags, 0),
},
&privateendpoints.PrivateEndpointSpec{
Name: "my-private-endpoint-2",
Namespace: "dummy-ns",
ResourceGroup: "dummy-rg",
Location: "westus2",
CustomNetworkInterfaceName: "my-custom-nic-2",
PrivateIPAddresses: []string{
"IP3",
"IP4",
},
SubnetID: "dummy-subnet-id",
ApplicationSecurityGroups: []string{
"ASG3",
"ASG4",
},
ClusterName: "my-cluster",
PrivateLinkServiceConnections: []privateendpoints.PrivateLinkServiceConnection{
{
Name: "my-pls-connection",
RequestMessage: "my-request-message",
PrivateLinkServiceID: "my-pls-id",
GroupIDs: []string{
"my-group-id-1",
},
},
},
AdditionalTags: make(infrav1.Tags, 0),
},
&privateendpoints.PrivateEndpointSpec{
Name: "my-private-endpoint-3",
Namespace: "dummy-ns",
ResourceGroup: "dummy-rg",
Location: "westus2",
CustomNetworkInterfaceName: "my-custom-nic-3",
PrivateIPAddresses: []string{
"IP5",
"IP6",
},
SubnetID: "dummy-subnet-id-2",
ApplicationSecurityGroups: []string{
"ASG5",
"ASG6",
},
ClusterName: "my-cluster",
PrivateLinkServiceConnections: []privateendpoints.PrivateLinkServiceConnection{
{
Name: "my-pls-connection",
RequestMessage: "my-request-message",
PrivateLinkServiceID: "my-pls-id",
GroupIDs: []string{
"my-group-id-1",
},
},
},
AdditionalTags: make(infrav1.Tags, 0),
},
},
},
}

for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
if got := tt.clusterScope.PrivateEndpointSpecs(); !reflect.DeepEqual(got, tt.want) {
t.Errorf("PrivateEndpointSpecs() = %s, want %s", specArrayToString(got), specArrayToString(tt.want))
}
})
}
}
7 changes: 4 additions & 3 deletions azure/scope/managedcontrolplane.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"strings"
"time"

asonetworkv1 "github.com/Azure/azure-service-operator/v2/api/network/v1api20220701"
asoresourcesv1 "github.com/Azure/azure-service-operator/v2/api/resources/v1api20200601"
"github.com/pkg/errors"
"golang.org/x/mod/semver"
Expand Down Expand Up @@ -807,12 +808,13 @@ func (s *ManagedControlPlaneScope) AvailabilityStatusFilter(cond *clusterv1.Cond
}

// PrivateEndpointSpecs returns the private endpoint specs.
func (s *ManagedControlPlaneScope) PrivateEndpointSpecs() []azure.ResourceSpecGetter {
privateEndpointSpecs := make([]azure.ResourceSpecGetter, len(s.ControlPlane.Spec.VirtualNetwork.Subnet.PrivateEndpoints))
func (s *ManagedControlPlaneScope) PrivateEndpointSpecs() []azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint] {
privateEndpointSpecs := make([]azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint], 0, len(s.ControlPlane.Spec.VirtualNetwork.Subnet.PrivateEndpoints))

for _, privateEndpoint := range s.ControlPlane.Spec.VirtualNetwork.Subnet.PrivateEndpoints {
privateEndpointSpec := &privateendpoints.PrivateEndpointSpec{
Name: privateEndpoint.Name,
Namespace: s.Cluster.Namespace,
ResourceGroup: s.VNetSpec().ResourceGroupName(),
Location: privateEndpoint.Location,
CustomNetworkInterfaceName: privateEndpoint.CustomNetworkInterfaceName,
Expand All @@ -838,7 +840,6 @@ func (s *ManagedControlPlaneScope) PrivateEndpointSpecs() []azure.ResourceSpecGe
}
privateEndpointSpec.PrivateLinkServiceConnections = append(privateEndpointSpec.PrivateLinkServiceConnections, pl)
}

privateEndpointSpecs = append(privateEndpointSpecs, privateEndpointSpec)
}

Expand Down
Loading

0 comments on commit 7c8f0d3

Please sign in to comment.