Skip to content

Commit

Permalink
Merge pull request #4108 from nawazkh/private_endpoints_to_aso
Browse files Browse the repository at this point in the history 
migrate private endpoints service to ASO
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot authored Dec 8, 2023
2 parents 9e6361a + 12a2c08 commit 7d7e883
Show file tree
Hide file tree
Showing 19 changed files with 1,757 additions and 1,147 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ WEBHOOK_ROOT ?= $(MANIFEST_ROOT)/webhook
RBAC_ROOT ?= $(MANIFEST_ROOT)/rbac
ASO_CRDS_PATH := $(MANIFEST_ROOT)/aso/crds.yaml
ASO_VERSION := v2.4.0
ASO_CRDS := resourcegroups.resources.azure.com natgateways.network.azure.com managedclusters.containerservice.azure.com managedclustersagentpools.containerservice.azure.com bastionhosts.network.azure.com
ASO_CRDS := resourcegroups.resources.azure.com natgateways.network.azure.com managedclusters.containerservice.azure.com managedclustersagentpools.containerservice.azure.com bastionhosts.network.azure.com privateendpoints.network.azure.com

# Allow overriding the imagePullPolicy
PULL_POLICY ?= Always
Expand Down
73 changes: 32 additions & 41 deletions azure/scope/cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1074,54 +1074,45 @@ func (s *ClusterScope) SetAnnotation(key, value string) {
}

// PrivateEndpointSpecs returns the private endpoint specs.
func (s *ClusterScope) PrivateEndpointSpecs() []azure.ResourceSpecGetter {
numberOfSubnets := len(s.AzureCluster.Spec.NetworkSpec.Subnets)
func (s *ClusterScope) PrivateEndpointSpecs() []azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint] {
subnetsList := s.AzureCluster.Spec.NetworkSpec.Subnets
numberOfSubnets := len(subnetsList)
if s.IsAzureBastionEnabled() {
subnetsList = append(subnetsList, s.AzureCluster.Spec.BastionSpec.AzureBastion.Subnet)
numberOfSubnets++
}

privateEndpointSpecs := make([]azure.ResourceSpecGetter, 0, numberOfSubnets)

subnets := s.AzureCluster.Spec.NetworkSpec.Subnets
if s.IsAzureBastionEnabled() {
subnets = append(subnets, s.AzureCluster.Spec.BastionSpec.AzureBastion.Subnet)
}

for _, subnet := range subnets {
privateEndpointSpecs = append(privateEndpointSpecs, s.getPrivateEndpoints(subnet)...)
}

return privateEndpointSpecs
}

func (s *ClusterScope) getPrivateEndpoints(subnet infrav1.SubnetSpec) []azure.ResourceSpecGetter {
privateEndpointSpecs := make([]azure.ResourceSpecGetter, 0)

for _, privateEndpoint := range subnet.PrivateEndpoints {
privateEndpointSpec := &privateendpoints.PrivateEndpointSpec{
Name: privateEndpoint.Name,
ResourceGroup: s.ResourceGroup(),
Location: privateEndpoint.Location,
CustomNetworkInterfaceName: privateEndpoint.CustomNetworkInterfaceName,
PrivateIPAddresses: privateEndpoint.PrivateIPAddresses,
SubnetID: subnet.ID,
ApplicationSecurityGroups: privateEndpoint.ApplicationSecurityGroups,
ManualApproval: privateEndpoint.ManualApproval,
ClusterName: s.ClusterName(),
AdditionalTags: s.AdditionalTags(),
}
// privateEndpointSpecs will be an empty list if no private endpoints were found.
// We pre-allocate the list to avoid unnecessary allocations during append.
privateEndpointSpecs := make([]azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint], 0, numberOfSubnets)

for _, subnet := range subnetsList {
for _, privateEndpoint := range subnet.PrivateEndpoints {
privateEndpointSpec := &privateendpoints.PrivateEndpointSpec{
Name: privateEndpoint.Name,
Namespace: s.Namespace(),
ResourceGroup: s.ResourceGroup(),
Location: privateEndpoint.Location,
CustomNetworkInterfaceName: privateEndpoint.CustomNetworkInterfaceName,
PrivateIPAddresses: privateEndpoint.PrivateIPAddresses,
SubnetID: subnet.ID,
ApplicationSecurityGroups: privateEndpoint.ApplicationSecurityGroups,
ManualApproval: privateEndpoint.ManualApproval,
ClusterName: s.ClusterName(),
AdditionalTags: s.AdditionalTags(),
}

for _, privateLinkServiceConnection := range privateEndpoint.PrivateLinkServiceConnections {
pl := privateendpoints.PrivateLinkServiceConnection{
PrivateLinkServiceID: privateLinkServiceConnection.PrivateLinkServiceID,
Name: privateLinkServiceConnection.Name,
RequestMessage: privateLinkServiceConnection.RequestMessage,
GroupIDs: privateLinkServiceConnection.GroupIDs,
for _, privateLinkServiceConnection := range privateEndpoint.PrivateLinkServiceConnections {
pl := privateendpoints.PrivateLinkServiceConnection{
PrivateLinkServiceID: privateLinkServiceConnection.PrivateLinkServiceID,
Name: privateLinkServiceConnection.Name,
RequestMessage: privateLinkServiceConnection.RequestMessage,
GroupIDs: privateLinkServiceConnection.GroupIDs,
}
privateEndpointSpec.PrivateLinkServiceConnections = append(privateEndpointSpec.PrivateLinkServiceConnections, pl)
}
privateEndpointSpec.PrivateLinkServiceConnections = append(privateEndpointSpec.PrivateLinkServiceConnections, pl)
privateEndpointSpecs = append(privateEndpointSpecs, privateEndpointSpec)
}

privateEndpointSpecs = append(privateEndpointSpecs, privateEndpointSpec)
}

return privateEndpointSpecs
Expand Down
244 changes: 244 additions & 0 deletions azure/scope/cluster_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (
"sigs.k8s.io/cluster-api-provider-azure/azure/services/bastionhosts"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/loadbalancers"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/natgateways"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/privateendpoints"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/publicips"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/routetables"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/securitygroups"
Expand Down Expand Up @@ -3388,6 +3389,249 @@ func TestVNetPeerings(t *testing.T) {
}
}

func TestPrivateEndpointSpecs(t *testing.T) {
tests := []struct {
name string
clusterScope ClusterScope
want []azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint]
}{
{
name: "returns empty private endpoints list if no subnets are specified",
clusterScope: ClusterScope{
AzureCluster: &infrav1.AzureCluster{
Spec: infrav1.AzureClusterSpec{
NetworkSpec: infrav1.NetworkSpec{
Subnets: infrav1.Subnets{},
},
},
},
cache: &ClusterCache{},
},
want: make([]azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint], 0),
},
{
name: "returns empty private endpoints list if no private endpoints are specified",
clusterScope: ClusterScope{
AzureCluster: &infrav1.AzureCluster{
Spec: infrav1.AzureClusterSpec{
NetworkSpec: infrav1.NetworkSpec{
Subnets: []infrav1.SubnetSpec{
{
SubnetClassSpec: infrav1.SubnetClassSpec{
PrivateEndpoints: infrav1.PrivateEndpoints{},
},
},
},
},
},
},
cache: &ClusterCache{},
},
want: make([]azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint], 0),
},
{
name: "returns list of private endpoint specs if private endpoints are specified",
clusterScope: ClusterScope{
Cluster: &clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "my-cluster",
Namespace: "dummy-ns",
},
},
AzureCluster: &infrav1.AzureCluster{
Spec: infrav1.AzureClusterSpec{
ResourceGroup: "dummy-rg",
NetworkSpec: infrav1.NetworkSpec{
Subnets: []infrav1.SubnetSpec{
{
ID: "dummy-subnet-id",
SubnetClassSpec: infrav1.SubnetClassSpec{
PrivateEndpoints: []infrav1.PrivateEndpointSpec{
{
Name: "my-private-endpoint",
Location: "westus2",
CustomNetworkInterfaceName: "my-custom-nic",
PrivateIPAddresses: []string{
"IP1",
"IP2",
},
ApplicationSecurityGroups: []string{
"ASG1",
"ASG2",
},
PrivateLinkServiceConnections: []infrav1.PrivateLinkServiceConnection{
{
Name: "my-pls-connection",
RequestMessage: "my-request-message",
PrivateLinkServiceID: "my-pls-id",
GroupIDs: []string{
"my-group-id-1",
},
},
},
},
{
Name: "my-private-endpoint-2",
Location: "westus2",
CustomNetworkInterfaceName: "my-custom-nic-2",
PrivateIPAddresses: []string{
"IP3",
"IP4",
},
ApplicationSecurityGroups: []string{
"ASG3",
"ASG4",
},
PrivateLinkServiceConnections: []infrav1.PrivateLinkServiceConnection{
{
Name: "my-pls-connection",
RequestMessage: "my-request-message",
PrivateLinkServiceID: "my-pls-id",
GroupIDs: []string{
"my-group-id-1",
},
},
},
},
},
},
},
{
ID: "dummy-subnet-id-2",
SubnetClassSpec: infrav1.SubnetClassSpec{
PrivateEndpoints: []infrav1.PrivateEndpointSpec{
{
Name: "my-private-endpoint-3",
Location: "westus2",
CustomNetworkInterfaceName: "my-custom-nic-3",
PrivateIPAddresses: []string{
"IP5",
"IP6",
},
ApplicationSecurityGroups: []string{
"ASG5",
"ASG6",
},
PrivateLinkServiceConnections: []infrav1.PrivateLinkServiceConnection{
{
Name: "my-pls-connection",
RequestMessage: "my-request-message",
PrivateLinkServiceID: "my-pls-id",
GroupIDs: []string{
"my-group-id-1",
},
},
},
},
},
},
},
},
},
},
},
cache: &ClusterCache{},
},
want: []azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint]{
&privateendpoints.PrivateEndpointSpec{
Name: "my-private-endpoint",
Namespace: "dummy-ns",
ResourceGroup: "dummy-rg",
Location: "westus2",
CustomNetworkInterfaceName: "my-custom-nic",
PrivateIPAddresses: []string{
"IP1",
"IP2",
},
SubnetID: "dummy-subnet-id",
ApplicationSecurityGroups: []string{
"ASG1",
"ASG2",
},
ClusterName: "my-cluster",
PrivateLinkServiceConnections: []privateendpoints.PrivateLinkServiceConnection{
{
Name: "my-pls-connection",
RequestMessage: "my-request-message",
PrivateLinkServiceID: "my-pls-id",
GroupIDs: []string{
"my-group-id-1",
},
},
},
AdditionalTags: make(infrav1.Tags, 0),
},
&privateendpoints.PrivateEndpointSpec{
Name: "my-private-endpoint-2",
Namespace: "dummy-ns",
ResourceGroup: "dummy-rg",
Location: "westus2",
CustomNetworkInterfaceName: "my-custom-nic-2",
PrivateIPAddresses: []string{
"IP3",
"IP4",
},
SubnetID: "dummy-subnet-id",
ApplicationSecurityGroups: []string{
"ASG3",
"ASG4",
},
ClusterName: "my-cluster",
PrivateLinkServiceConnections: []privateendpoints.PrivateLinkServiceConnection{
{
Name: "my-pls-connection",
RequestMessage: "my-request-message",
PrivateLinkServiceID: "my-pls-id",
GroupIDs: []string{
"my-group-id-1",
},
},
},
AdditionalTags: make(infrav1.Tags, 0),
},
&privateendpoints.PrivateEndpointSpec{
Name: "my-private-endpoint-3",
Namespace: "dummy-ns",
ResourceGroup: "dummy-rg",
Location: "westus2",
CustomNetworkInterfaceName: "my-custom-nic-3",
PrivateIPAddresses: []string{
"IP5",
"IP6",
},
SubnetID: "dummy-subnet-id-2",
ApplicationSecurityGroups: []string{
"ASG5",
"ASG6",
},
ClusterName: "my-cluster",
PrivateLinkServiceConnections: []privateendpoints.PrivateLinkServiceConnection{
{
Name: "my-pls-connection",
RequestMessage: "my-request-message",
PrivateLinkServiceID: "my-pls-id",
GroupIDs: []string{
"my-group-id-1",
},
},
},
AdditionalTags: make(infrav1.Tags, 0),
},
},
},
}

for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
if got := tt.clusterScope.PrivateEndpointSpecs(); !reflect.DeepEqual(got, tt.want) {
t.Errorf("PrivateEndpointSpecs() = %s, want %s", specArrayToString(got), specArrayToString(tt.want))
}
})
}
}

func TestSetFailureDomain(t *testing.T) {
t.Parallel()

Expand Down
7 changes: 4 additions & 3 deletions azure/scope/managedcontrolplane.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"time"

asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20230201"
asonetworkv1 "github.com/Azure/azure-service-operator/v2/api/network/v1api20220701"
asoresourcesv1 "github.com/Azure/azure-service-operator/v2/api/resources/v1api20200601"
"github.com/pkg/errors"
"golang.org/x/mod/semver"
Expand Down Expand Up @@ -860,12 +861,13 @@ func (s *ManagedControlPlaneScope) AvailabilityStatusFilter(cond *clusterv1.Cond
}

// PrivateEndpointSpecs returns the private endpoint specs.
func (s *ManagedControlPlaneScope) PrivateEndpointSpecs() []azure.ResourceSpecGetter {
privateEndpointSpecs := make([]azure.ResourceSpecGetter, len(s.ControlPlane.Spec.VirtualNetwork.Subnet.PrivateEndpoints))
func (s *ManagedControlPlaneScope) PrivateEndpointSpecs() []azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint] {
privateEndpointSpecs := make([]azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint], 0, len(s.ControlPlane.Spec.VirtualNetwork.Subnet.PrivateEndpoints))

for _, privateEndpoint := range s.ControlPlane.Spec.VirtualNetwork.Subnet.PrivateEndpoints {
privateEndpointSpec := &privateendpoints.PrivateEndpointSpec{
Name: privateEndpoint.Name,
Namespace: s.Cluster.Namespace,
ResourceGroup: s.VNetSpec().ResourceGroupName(),
Location: privateEndpoint.Location,
CustomNetworkInterfaceName: privateEndpoint.CustomNetworkInterfaceName,
Expand All @@ -891,7 +893,6 @@ func (s *ManagedControlPlaneScope) PrivateEndpointSpecs() []azure.ResourceSpecGe
}
privateEndpointSpec.PrivateLinkServiceConnections = append(privateEndpointSpec.PrivateLinkServiceConnections, pl)
}

privateEndpointSpecs = append(privateEndpointSpecs, privateEndpointSpec)
}

Expand Down
Loading

0 comments on commit 7d7e883

Please sign in to comment.