From 12a2c08e03e23baaf3434b38137eda64c49f8222 Mon Sep 17 00:00:00 2001 From: Nawaz Hussain Khazielakha Date: Mon, 9 Oct 2023 15:00:11 -0500 Subject: [PATCH] migrate private endpoints service to use ASO framework CAPZ sets the following ASO fields if found: - ApplicationSecurityGroups - AzureName - CustomNetworkInterfaceName - IpConfigurations - Location - ManualPrivateLinkServiceConnections - Owner - PrivateLinkServiceConnections - Subnet - Tags ASO fields not managed by CAPZ - ExtendedLocation --- Makefile | 2 +- azure/scope/cluster.go | 73 +- azure/scope/cluster_test.go | 244 +++++ azure/scope/managedcontrolplane.go | 7 +- azure/scope/managedcontrolplane_test.go | 177 ++++ azure/services/privateendpoints/client.go | 120 --- .../mock_privateendpoints/client_mock.go | 25 - .../mock_privateendpoints/doc.go | 2 - .../privateendpoints_mock.go | 129 +-- .../privateendpoints/privateendpoints.go | 107 +- .../privateendpoints/privateendpoints_test.go | 258 ----- azure/services/privateendpoints/spec.go | 267 ++--- azure/services/privateendpoints/spec_test.go | 463 +++----- config/aso/crds.yaml | 990 ++++++++++++++++++ config/rbac/role.yaml | 22 + controllers/azurecluster_controller.go | 4 +- controllers/azurecluster_reconciler.go | 6 +- .../azuremanagedcontrolplane_controller.go | 2 + .../azuremanagedcontrolplane_reconciler.go | 6 +- 19 files changed, 1757 insertions(+), 1147 deletions(-) delete mode 100644 azure/services/privateendpoints/client.go delete mode 100644 azure/services/privateendpoints/mock_privateendpoints/client_mock.go delete mode 100644 azure/services/privateendpoints/privateendpoints_test.go diff --git a/Makefile b/Makefile index eb524b844c6..6f2666f7c60 100644 --- a/Makefile +++ b/Makefile @@ -158,7 +158,7 @@ WEBHOOK_ROOT ?= $(MANIFEST_ROOT)/webhook RBAC_ROOT ?= $(MANIFEST_ROOT)/rbac ASO_CRDS_PATH := $(MANIFEST_ROOT)/aso/crds.yaml ASO_VERSION := v2.4.0 -ASO_CRDS := resourcegroups.resources.azure.com natgateways.network.azure.com managedclusters.containerservice.azure.com managedclustersagentpools.containerservice.azure.com bastionhosts.network.azure.com +ASO_CRDS := resourcegroups.resources.azure.com natgateways.network.azure.com managedclusters.containerservice.azure.com managedclustersagentpools.containerservice.azure.com bastionhosts.network.azure.com privateendpoints.network.azure.com # Allow overriding the imagePullPolicy PULL_POLICY ?= Always diff --git a/azure/scope/cluster.go b/azure/scope/cluster.go index 8ea909ee438..b7ce32f0cd7 100644 --- a/azure/scope/cluster.go +++ b/azure/scope/cluster.go @@ -1074,54 +1074,45 @@ func (s *ClusterScope) SetAnnotation(key, value string) { } // PrivateEndpointSpecs returns the private endpoint specs. -func (s *ClusterScope) PrivateEndpointSpecs() []azure.ResourceSpecGetter { - numberOfSubnets := len(s.AzureCluster.Spec.NetworkSpec.Subnets) +func (s *ClusterScope) PrivateEndpointSpecs() []azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint] { + subnetsList := s.AzureCluster.Spec.NetworkSpec.Subnets + numberOfSubnets := len(subnetsList) if s.IsAzureBastionEnabled() { + subnetsList = append(subnetsList, s.AzureCluster.Spec.BastionSpec.AzureBastion.Subnet) numberOfSubnets++ } - privateEndpointSpecs := make([]azure.ResourceSpecGetter, 0, numberOfSubnets) - - subnets := s.AzureCluster.Spec.NetworkSpec.Subnets - if s.IsAzureBastionEnabled() { - subnets = append(subnets, s.AzureCluster.Spec.BastionSpec.AzureBastion.Subnet) - } - - for _, subnet := range subnets { - privateEndpointSpecs = append(privateEndpointSpecs, s.getPrivateEndpoints(subnet)...) - } - - return privateEndpointSpecs -} - -func (s *ClusterScope) getPrivateEndpoints(subnet infrav1.SubnetSpec) []azure.ResourceSpecGetter { - privateEndpointSpecs := make([]azure.ResourceSpecGetter, 0) - - for _, privateEndpoint := range subnet.PrivateEndpoints { - privateEndpointSpec := &privateendpoints.PrivateEndpointSpec{ - Name: privateEndpoint.Name, - ResourceGroup: s.ResourceGroup(), - Location: privateEndpoint.Location, - CustomNetworkInterfaceName: privateEndpoint.CustomNetworkInterfaceName, - PrivateIPAddresses: privateEndpoint.PrivateIPAddresses, - SubnetID: subnet.ID, - ApplicationSecurityGroups: privateEndpoint.ApplicationSecurityGroups, - ManualApproval: privateEndpoint.ManualApproval, - ClusterName: s.ClusterName(), - AdditionalTags: s.AdditionalTags(), - } + // privateEndpointSpecs will be an empty list if no private endpoints were found. + // We pre-allocate the list to avoid unnecessary allocations during append. + privateEndpointSpecs := make([]azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint], 0, numberOfSubnets) + + for _, subnet := range subnetsList { + for _, privateEndpoint := range subnet.PrivateEndpoints { + privateEndpointSpec := &privateendpoints.PrivateEndpointSpec{ + Name: privateEndpoint.Name, + Namespace: s.Namespace(), + ResourceGroup: s.ResourceGroup(), + Location: privateEndpoint.Location, + CustomNetworkInterfaceName: privateEndpoint.CustomNetworkInterfaceName, + PrivateIPAddresses: privateEndpoint.PrivateIPAddresses, + SubnetID: subnet.ID, + ApplicationSecurityGroups: privateEndpoint.ApplicationSecurityGroups, + ManualApproval: privateEndpoint.ManualApproval, + ClusterName: s.ClusterName(), + AdditionalTags: s.AdditionalTags(), + } - for _, privateLinkServiceConnection := range privateEndpoint.PrivateLinkServiceConnections { - pl := privateendpoints.PrivateLinkServiceConnection{ - PrivateLinkServiceID: privateLinkServiceConnection.PrivateLinkServiceID, - Name: privateLinkServiceConnection.Name, - RequestMessage: privateLinkServiceConnection.RequestMessage, - GroupIDs: privateLinkServiceConnection.GroupIDs, + for _, privateLinkServiceConnection := range privateEndpoint.PrivateLinkServiceConnections { + pl := privateendpoints.PrivateLinkServiceConnection{ + PrivateLinkServiceID: privateLinkServiceConnection.PrivateLinkServiceID, + Name: privateLinkServiceConnection.Name, + RequestMessage: privateLinkServiceConnection.RequestMessage, + GroupIDs: privateLinkServiceConnection.GroupIDs, + } + privateEndpointSpec.PrivateLinkServiceConnections = append(privateEndpointSpec.PrivateLinkServiceConnections, pl) } - privateEndpointSpec.PrivateLinkServiceConnections = append(privateEndpointSpec.PrivateLinkServiceConnections, pl) + privateEndpointSpecs = append(privateEndpointSpecs, privateEndpointSpec) } - - privateEndpointSpecs = append(privateEndpointSpecs, privateEndpointSpec) } return privateEndpointSpecs diff --git a/azure/scope/cluster_test.go b/azure/scope/cluster_test.go index 4abe02bbd05..35908f8c091 100644 --- a/azure/scope/cluster_test.go +++ b/azure/scope/cluster_test.go @@ -35,6 +35,7 @@ import ( "sigs.k8s.io/cluster-api-provider-azure/azure/services/bastionhosts" "sigs.k8s.io/cluster-api-provider-azure/azure/services/loadbalancers" "sigs.k8s.io/cluster-api-provider-azure/azure/services/natgateways" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/privateendpoints" "sigs.k8s.io/cluster-api-provider-azure/azure/services/publicips" "sigs.k8s.io/cluster-api-provider-azure/azure/services/routetables" "sigs.k8s.io/cluster-api-provider-azure/azure/services/securitygroups" @@ -3388,6 +3389,249 @@ func TestVNetPeerings(t *testing.T) { } } +func TestPrivateEndpointSpecs(t *testing.T) { + tests := []struct { + name string + clusterScope ClusterScope + want []azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint] + }{ + { + name: "returns empty private endpoints list if no subnets are specified", + clusterScope: ClusterScope{ + AzureCluster: &infrav1.AzureCluster{ + Spec: infrav1.AzureClusterSpec{ + NetworkSpec: infrav1.NetworkSpec{ + Subnets: infrav1.Subnets{}, + }, + }, + }, + cache: &ClusterCache{}, + }, + want: make([]azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint], 0), + }, + { + name: "returns empty private endpoints list if no private endpoints are specified", + clusterScope: ClusterScope{ + AzureCluster: &infrav1.AzureCluster{ + Spec: infrav1.AzureClusterSpec{ + NetworkSpec: infrav1.NetworkSpec{ + Subnets: []infrav1.SubnetSpec{ + { + SubnetClassSpec: infrav1.SubnetClassSpec{ + PrivateEndpoints: infrav1.PrivateEndpoints{}, + }, + }, + }, + }, + }, + }, + cache: &ClusterCache{}, + }, + want: make([]azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint], 0), + }, + { + name: "returns list of private endpoint specs if private endpoints are specified", + clusterScope: ClusterScope{ + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster", + Namespace: "dummy-ns", + }, + }, + AzureCluster: &infrav1.AzureCluster{ + Spec: infrav1.AzureClusterSpec{ + ResourceGroup: "dummy-rg", + NetworkSpec: infrav1.NetworkSpec{ + Subnets: []infrav1.SubnetSpec{ + { + ID: "dummy-subnet-id", + SubnetClassSpec: infrav1.SubnetClassSpec{ + PrivateEndpoints: []infrav1.PrivateEndpointSpec{ + { + Name: "my-private-endpoint", + Location: "westus2", + CustomNetworkInterfaceName: "my-custom-nic", + PrivateIPAddresses: []string{ + "IP1", + "IP2", + }, + ApplicationSecurityGroups: []string{ + "ASG1", + "ASG2", + }, + PrivateLinkServiceConnections: []infrav1.PrivateLinkServiceConnection{ + { + Name: "my-pls-connection", + RequestMessage: "my-request-message", + PrivateLinkServiceID: "my-pls-id", + GroupIDs: []string{ + "my-group-id-1", + }, + }, + }, + }, + { + Name: "my-private-endpoint-2", + Location: "westus2", + CustomNetworkInterfaceName: "my-custom-nic-2", + PrivateIPAddresses: []string{ + "IP3", + "IP4", + }, + ApplicationSecurityGroups: []string{ + "ASG3", + "ASG4", + }, + PrivateLinkServiceConnections: []infrav1.PrivateLinkServiceConnection{ + { + Name: "my-pls-connection", + RequestMessage: "my-request-message", + PrivateLinkServiceID: "my-pls-id", + GroupIDs: []string{ + "my-group-id-1", + }, + }, + }, + }, + }, + }, + }, + { + ID: "dummy-subnet-id-2", + SubnetClassSpec: infrav1.SubnetClassSpec{ + PrivateEndpoints: []infrav1.PrivateEndpointSpec{ + { + Name: "my-private-endpoint-3", + Location: "westus2", + CustomNetworkInterfaceName: "my-custom-nic-3", + PrivateIPAddresses: []string{ + "IP5", + "IP6", + }, + ApplicationSecurityGroups: []string{ + "ASG5", + "ASG6", + }, + PrivateLinkServiceConnections: []infrav1.PrivateLinkServiceConnection{ + { + Name: "my-pls-connection", + RequestMessage: "my-request-message", + PrivateLinkServiceID: "my-pls-id", + GroupIDs: []string{ + "my-group-id-1", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + cache: &ClusterCache{}, + }, + want: []azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint]{ + &privateendpoints.PrivateEndpointSpec{ + Name: "my-private-endpoint", + Namespace: "dummy-ns", + ResourceGroup: "dummy-rg", + Location: "westus2", + CustomNetworkInterfaceName: "my-custom-nic", + PrivateIPAddresses: []string{ + "IP1", + "IP2", + }, + SubnetID: "dummy-subnet-id", + ApplicationSecurityGroups: []string{ + "ASG1", + "ASG2", + }, + ClusterName: "my-cluster", + PrivateLinkServiceConnections: []privateendpoints.PrivateLinkServiceConnection{ + { + Name: "my-pls-connection", + RequestMessage: "my-request-message", + PrivateLinkServiceID: "my-pls-id", + GroupIDs: []string{ + "my-group-id-1", + }, + }, + }, + AdditionalTags: make(infrav1.Tags, 0), + }, + &privateendpoints.PrivateEndpointSpec{ + Name: "my-private-endpoint-2", + Namespace: "dummy-ns", + ResourceGroup: "dummy-rg", + Location: "westus2", + CustomNetworkInterfaceName: "my-custom-nic-2", + PrivateIPAddresses: []string{ + "IP3", + "IP4", + }, + SubnetID: "dummy-subnet-id", + ApplicationSecurityGroups: []string{ + "ASG3", + "ASG4", + }, + ClusterName: "my-cluster", + PrivateLinkServiceConnections: []privateendpoints.PrivateLinkServiceConnection{ + { + Name: "my-pls-connection", + RequestMessage: "my-request-message", + PrivateLinkServiceID: "my-pls-id", + GroupIDs: []string{ + "my-group-id-1", + }, + }, + }, + AdditionalTags: make(infrav1.Tags, 0), + }, + &privateendpoints.PrivateEndpointSpec{ + Name: "my-private-endpoint-3", + Namespace: "dummy-ns", + ResourceGroup: "dummy-rg", + Location: "westus2", + CustomNetworkInterfaceName: "my-custom-nic-3", + PrivateIPAddresses: []string{ + "IP5", + "IP6", + }, + SubnetID: "dummy-subnet-id-2", + ApplicationSecurityGroups: []string{ + "ASG5", + "ASG6", + }, + ClusterName: "my-cluster", + PrivateLinkServiceConnections: []privateendpoints.PrivateLinkServiceConnection{ + { + Name: "my-pls-connection", + RequestMessage: "my-request-message", + PrivateLinkServiceID: "my-pls-id", + GroupIDs: []string{ + "my-group-id-1", + }, + }, + }, + AdditionalTags: make(infrav1.Tags, 0), + }, + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := tt.clusterScope.PrivateEndpointSpecs(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("PrivateEndpointSpecs() = %s, want %s", specArrayToString(got), specArrayToString(tt.want)) + } + }) + } +} + func TestSetFailureDomain(t *testing.T) { t.Parallel() diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go index 9189d1d1988..3f92ea9ef80 100644 --- a/azure/scope/managedcontrolplane.go +++ b/azure/scope/managedcontrolplane.go @@ -24,6 +24,7 @@ import ( "time" asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20230201" + asonetworkv1 "github.com/Azure/azure-service-operator/v2/api/network/v1api20220701" asoresourcesv1 "github.com/Azure/azure-service-operator/v2/api/resources/v1api20200601" "github.com/pkg/errors" "golang.org/x/mod/semver" @@ -860,12 +861,13 @@ func (s *ManagedControlPlaneScope) AvailabilityStatusFilter(cond *clusterv1.Cond } // PrivateEndpointSpecs returns the private endpoint specs. -func (s *ManagedControlPlaneScope) PrivateEndpointSpecs() []azure.ResourceSpecGetter { - privateEndpointSpecs := make([]azure.ResourceSpecGetter, len(s.ControlPlane.Spec.VirtualNetwork.Subnet.PrivateEndpoints)) +func (s *ManagedControlPlaneScope) PrivateEndpointSpecs() []azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint] { + privateEndpointSpecs := make([]azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint], 0, len(s.ControlPlane.Spec.VirtualNetwork.Subnet.PrivateEndpoints)) for _, privateEndpoint := range s.ControlPlane.Spec.VirtualNetwork.Subnet.PrivateEndpoints { privateEndpointSpec := &privateendpoints.PrivateEndpointSpec{ Name: privateEndpoint.Name, + Namespace: s.Cluster.Namespace, ResourceGroup: s.VNetSpec().ResourceGroupName(), Location: privateEndpoint.Location, CustomNetworkInterfaceName: privateEndpoint.CustomNetworkInterfaceName, @@ -891,7 +893,6 @@ func (s *ManagedControlPlaneScope) PrivateEndpointSpecs() []azure.ResourceSpecGe } privateEndpointSpec.PrivateLinkServiceConnections = append(privateEndpointSpec.PrivateLinkServiceConnections, pl) } - privateEndpointSpecs = append(privateEndpointSpecs, privateEndpointSpec) } diff --git a/azure/scope/managedcontrolplane_test.go b/azure/scope/managedcontrolplane_test.go index 014f0f06e5a..f4c9b7df62b 100644 --- a/azure/scope/managedcontrolplane_test.go +++ b/azure/scope/managedcontrolplane_test.go @@ -18,9 +18,12 @@ package scope import ( "context" + "reflect" "testing" asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20230201" + asonetworkv1 "github.com/Azure/azure-service-operator/v2/api/network/v1api20220701" + "github.com/Azure/go-autorest/autorest" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -29,6 +32,7 @@ import ( "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/services/agentpools" "sigs.k8s.io/cluster-api-provider-azure/azure/services/managedclusters" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/privateendpoints" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -1027,3 +1031,176 @@ func TestAreLocalAccountsDisabled(t *testing.T) { }) } } + +func TestManagedControlPlaneScope_PrivateEndpointSpecs(t *testing.T) { + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected []azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint] + Err string + }{ + { + Name: "returns empty private endpoints list if no subnets are specified", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + VirtualNetwork: infrav1.ManagedControlPlaneVirtualNetwork{}, + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + }, + }, + Expected: make([]azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint], 0), + }, + { + Name: "returns empty private endpoints list if no private endpoints are specified", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + VirtualNetwork: infrav1.ManagedControlPlaneVirtualNetwork{ + ManagedControlPlaneVirtualNetworkClassSpec: infrav1.ManagedControlPlaneVirtualNetworkClassSpec{ + Subnet: infrav1.ManagedControlPlaneSubnet{ + PrivateEndpoints: infrav1.PrivateEndpoints{}, + }, + }, + }, + }, + }, + }, + }, + Expected: make([]azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint], 0), + }, + { + Name: "returns list of private endpoint specs if private endpoints are specified", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster", + Namespace: "dummy-ns", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster", + Namespace: "dummy-ns", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + AzureManagedControlPlaneClassSpec: infrav1.AzureManagedControlPlaneClassSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000001", + VirtualNetwork: infrav1.ManagedControlPlaneVirtualNetwork{ + ResourceGroup: "dummy-rg", + ManagedControlPlaneVirtualNetworkClassSpec: infrav1.ManagedControlPlaneVirtualNetworkClassSpec{ + Name: "vnet1", + Subnet: infrav1.ManagedControlPlaneSubnet{ + Name: "subnet1", + PrivateEndpoints: infrav1.PrivateEndpoints{ + { + Name: "my-private-endpoint", + Location: "westus2", + PrivateLinkServiceConnections: []infrav1.PrivateLinkServiceConnection{ + { + Name: "my-pls-connection", + PrivateLinkServiceID: "my-pls-id", + GroupIDs: []string{ + "my-group-id-1", + }, + RequestMessage: "my-request-message", + }, + }, + CustomNetworkInterfaceName: "my-custom-nic", + PrivateIPAddresses: []string{ + "IP1", + "IP2", + }, + ApplicationSecurityGroups: []string{ + "ASG1", + "ASG2", + }, + ManualApproval: true, + }, + }, + }, + }, + }, + }, + }, + }, + }, + Expected: []azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint]{ + &privateendpoints.PrivateEndpointSpec{ + Name: "my-private-endpoint", + Namespace: "dummy-ns", + ResourceGroup: "dummy-rg", + Location: "westus2", + CustomNetworkInterfaceName: "my-custom-nic", + PrivateIPAddresses: []string{ + "IP1", + "IP2", + }, + SubnetID: "/subscriptions/00000000-0000-0000-0000-000000000001/resourceGroups/dummy-rg/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/subnet1", + ApplicationSecurityGroups: []string{ + "ASG1", + "ASG2", + }, + ClusterName: "my-cluster", + PrivateLinkServiceConnections: []privateendpoints.PrivateLinkServiceConnection{ + { + Name: "my-pls-connection", + RequestMessage: "my-request-message", + PrivateLinkServiceID: "my-pls-id", + GroupIDs: []string{ + "my-group-id-1", + }, + }, + }, + ManualApproval: true, + AdditionalTags: make(infrav1.Tags, 0), + }, + }, + }, + } + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + s := &ManagedControlPlaneScope{ + ControlPlane: c.Input.ControlPlane, + Cluster: c.Input.Cluster, + } + if got := s.PrivateEndpointSpecs(); !reflect.DeepEqual(got, c.Expected) { + t.Errorf("PrivateEndpointSpecs() = %s, want %s", specArrayToString(got), specArrayToString(c.Expected)) + } + }) + } +} diff --git a/azure/services/privateendpoints/client.go b/azure/services/privateendpoints/client.go deleted file mode 100644 index 353a4a3102a..00000000000 --- a/azure/services/privateendpoints/client.go +++ /dev/null @@ -1,120 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package privateendpoints - -import ( - "context" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4" - "github.com/pkg/errors" - "sigs.k8s.io/cluster-api-provider-azure/azure" - "sigs.k8s.io/cluster-api-provider-azure/azure/services/async" - "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" - "sigs.k8s.io/cluster-api-provider-azure/util/tele" -) - -// azureClient contains the Azure go-sdk Client. -type azureClient struct { - privateendpoints *armnetwork.PrivateEndpointsClient -} - -// newClient creates a new private endpoint client from an authorizer. -func newClient(auth azure.Authorizer) (*azureClient, error) { - opts, err := azure.ARMClientOptions(auth.CloudEnvironment()) - if err != nil { - return nil, errors.Wrap(err, "failed to create privateendpoints client options") - } - factory, err := armnetwork.NewClientFactory(auth.SubscriptionID(), auth.Token(), opts) - if err != nil { - return nil, errors.Wrap(err, "failed to create armnetwork client factory") - } - return &azureClient{factory.NewPrivateEndpointsClient()}, nil -} - -// Get gets the specified private endpoint by the private endpoint name. -func (ac *azureClient) Get(ctx context.Context, spec azure.ResourceSpecGetter) (result interface{}, err error) { - ctx, _, done := tele.StartSpanWithLogger(ctx, "privateendpoints.azureClient.Get") - defer done() - - resp, err := ac.privateendpoints.Get(ctx, spec.ResourceGroupName(), spec.ResourceName(), nil) - if err != nil { - return nil, err - } - return resp.PrivateEndpoint, nil -} - -// CreateOrUpdateAsync creates a private endpoint. -// It sends a PUT request to Azure and if accepted without error, the func will return a Poller which can be used to track the ongoing -// progress of the operation. -func (ac *azureClient) CreateOrUpdateAsync(ctx context.Context, spec azure.ResourceSpecGetter, resumeToken string, parameters interface{}) (result interface{}, poller *runtime.Poller[armnetwork.PrivateEndpointsClientCreateOrUpdateResponse], err error) { - ctx, _, done := tele.StartSpanWithLogger(ctx, "privateendpoints.azureClient.CreateOrUpdateAsync") - defer done() - - pe, ok := parameters.(armnetwork.PrivateEndpoint) - if !ok && parameters != nil { - return nil, nil, errors.Errorf("%T is not an armnetwork.PrivateEndpoint", parameters) - } - - opts := &armnetwork.PrivateEndpointsClientBeginCreateOrUpdateOptions{ResumeToken: resumeToken} - poller, err = ac.privateendpoints.BeginCreateOrUpdate(ctx, spec.ResourceGroupName(), spec.ResourceName(), pe, opts) - if err != nil { - return nil, nil, err - } - - ctx, cancel := context.WithTimeout(ctx, reconciler.DefaultAzureCallTimeout) - defer cancel() - - pollOpts := &runtime.PollUntilDoneOptions{Frequency: async.DefaultPollerFrequency} - resp, err := poller.PollUntilDone(ctx, pollOpts) - if err != nil { - // if an error occurs, return the poller. - // this means the long-running operation didn't finish in the specified timeout. - return nil, poller, err - } - - // if the operation completed, return a nil poller - return resp.PrivateEndpoint, nil, err -} - -// DeleteAsync deletes a private endpoint asynchronously. DeleteAsync sends a DELETE -// request to Azure and if accepted without error, the func will return a Poller which can be used to track the ongoing -// progress of the operation. -func (ac *azureClient) DeleteAsync(ctx context.Context, spec azure.ResourceSpecGetter, resumeToken string) (poller *runtime.Poller[armnetwork.PrivateEndpointsClientDeleteResponse], err error) { - ctx, _, done := tele.StartSpanWithLogger(ctx, "privateendpoints.azureClient.DeleteAsync") - defer done() - - opts := &armnetwork.PrivateEndpointsClientBeginDeleteOptions{ResumeToken: resumeToken} - poller, err = ac.privateendpoints.BeginDelete(ctx, spec.ResourceGroupName(), spec.ResourceName(), opts) - if err != nil { - return nil, err - } - - ctx, cancel := context.WithTimeout(ctx, reconciler.DefaultAzureCallTimeout) - defer cancel() - - pollOpts := &runtime.PollUntilDoneOptions{Frequency: async.DefaultPollerFrequency} - _, err = poller.PollUntilDone(ctx, pollOpts) - if err != nil { - // if an error occurs, return the poller. - // this means the long-running operation didn't finish in the specified timeout. - return poller, err - } - - // if the operation completed, return a nil poller. - return nil, err -} diff --git a/azure/services/privateendpoints/mock_privateendpoints/client_mock.go b/azure/services/privateendpoints/mock_privateendpoints/client_mock.go deleted file mode 100644 index 00e9d1ba2ff..00000000000 --- a/azure/services/privateendpoints/mock_privateendpoints/client_mock.go +++ /dev/null @@ -1,25 +0,0 @@ -/* -Copyright The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by MockGen. DO NOT EDIT. -// Source: ../client.go -// -// Generated by this command: -// -// mockgen -destination client_mock.go -package mock_privateendpoints -source ../client.go Client -// -// Package mock_privateendpoints is a generated GoMock package. -package mock_privateendpoints diff --git a/azure/services/privateendpoints/mock_privateendpoints/doc.go b/azure/services/privateendpoints/mock_privateendpoints/doc.go index 9eb70fd39fc..6972354f34f 100644 --- a/azure/services/privateendpoints/mock_privateendpoints/doc.go +++ b/azure/services/privateendpoints/mock_privateendpoints/doc.go @@ -16,8 +16,6 @@ limitations under the License. // Run go generate to regenerate this mock. // -//go:generate ../../../../hack/tools/bin/mockgen -destination client_mock.go -package mock_privateendpoints -source ../client.go Client //go:generate ../../../../hack/tools/bin/mockgen -destination privateendpoints_mock.go -package mock_privateendpoints -source ../privateendpoints.go PrivateEndpointScope -//go:generate /usr/bin/env bash -c "cat ../../../../hack/boilerplate/boilerplate.generatego.txt client_mock.go > _client_mock.go && mv _client_mock.go client_mock.go" //go:generate /usr/bin/env bash -c "cat ../../../../hack/boilerplate/boilerplate.generatego.txt privateendpoints_mock.go > _privateendpoints_mock.go && mv _privateendpoints_mock.go privateendpoints_mock.go" package mock_privateendpoints diff --git a/azure/services/privateendpoints/mock_privateendpoints/privateendpoints_mock.go b/azure/services/privateendpoints/mock_privateendpoints/privateendpoints_mock.go index b5db26a8e3d..faea62448b9 100644 --- a/azure/services/privateendpoints/mock_privateendpoints/privateendpoints_mock.go +++ b/azure/services/privateendpoints/mock_privateendpoints/privateendpoints_mock.go @@ -27,11 +27,12 @@ package mock_privateendpoints import ( reflect "reflect" - azcore "github.com/Azure/azure-sdk-for-go/sdk/azcore" + v1api20220701 "github.com/Azure/azure-service-operator/v2/api/network/v1api20220701" gomock "go.uber.org/mock/gomock" v1beta1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" azure "sigs.k8s.io/cluster-api-provider-azure/azure" v1beta10 "sigs.k8s.io/cluster-api/api/v1beta1" + client "sigs.k8s.io/controller-runtime/pkg/client" ) // MockPrivateEndpointScope is a mock of PrivateEndpointScope interface. @@ -57,72 +58,44 @@ func (m *MockPrivateEndpointScope) EXPECT() *MockPrivateEndpointScopeMockRecorde return m.recorder } -// BaseURI mocks base method. -func (m *MockPrivateEndpointScope) BaseURI() string { +// ClusterName mocks base method. +func (m *MockPrivateEndpointScope) ClusterName() string { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "BaseURI") + ret := m.ctrl.Call(m, "ClusterName") ret0, _ := ret[0].(string) return ret0 } -// BaseURI indicates an expected call of BaseURI. -func (mr *MockPrivateEndpointScopeMockRecorder) BaseURI() *gomock.Call { +// ClusterName indicates an expected call of ClusterName. +func (mr *MockPrivateEndpointScopeMockRecorder) ClusterName() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BaseURI", reflect.TypeOf((*MockPrivateEndpointScope)(nil).BaseURI)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClusterName", reflect.TypeOf((*MockPrivateEndpointScope)(nil).ClusterName)) } -// ClientID mocks base method. -func (m *MockPrivateEndpointScope) ClientID() string { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ClientID") - ret0, _ := ret[0].(string) - return ret0 -} - -// ClientID indicates an expected call of ClientID. -func (mr *MockPrivateEndpointScopeMockRecorder) ClientID() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientID", reflect.TypeOf((*MockPrivateEndpointScope)(nil).ClientID)) -} - -// ClientSecret mocks base method. -func (m *MockPrivateEndpointScope) ClientSecret() string { +// DeleteLongRunningOperationState mocks base method. +func (m *MockPrivateEndpointScope) DeleteLongRunningOperationState(arg0, arg1, arg2 string) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ClientSecret") - ret0, _ := ret[0].(string) - return ret0 + m.ctrl.Call(m, "DeleteLongRunningOperationState", arg0, arg1, arg2) } -// ClientSecret indicates an expected call of ClientSecret. -func (mr *MockPrivateEndpointScopeMockRecorder) ClientSecret() *gomock.Call { +// DeleteLongRunningOperationState indicates an expected call of DeleteLongRunningOperationState. +func (mr *MockPrivateEndpointScopeMockRecorder) DeleteLongRunningOperationState(arg0, arg1, arg2 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientSecret", reflect.TypeOf((*MockPrivateEndpointScope)(nil).ClientSecret)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLongRunningOperationState", reflect.TypeOf((*MockPrivateEndpointScope)(nil).DeleteLongRunningOperationState), arg0, arg1, arg2) } -// CloudEnvironment mocks base method. -func (m *MockPrivateEndpointScope) CloudEnvironment() string { +// GetClient mocks base method. +func (m *MockPrivateEndpointScope) GetClient() client.Client { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CloudEnvironment") - ret0, _ := ret[0].(string) + ret := m.ctrl.Call(m, "GetClient") + ret0, _ := ret[0].(client.Client) return ret0 } -// CloudEnvironment indicates an expected call of CloudEnvironment. -func (mr *MockPrivateEndpointScopeMockRecorder) CloudEnvironment() *gomock.Call { +// GetClient indicates an expected call of GetClient. +func (mr *MockPrivateEndpointScopeMockRecorder) GetClient() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CloudEnvironment", reflect.TypeOf((*MockPrivateEndpointScope)(nil).CloudEnvironment)) -} - -// DeleteLongRunningOperationState mocks base method. -func (m *MockPrivateEndpointScope) DeleteLongRunningOperationState(arg0, arg1, arg2 string) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "DeleteLongRunningOperationState", arg0, arg1, arg2) -} - -// DeleteLongRunningOperationState indicates an expected call of DeleteLongRunningOperationState. -func (mr *MockPrivateEndpointScopeMockRecorder) DeleteLongRunningOperationState(arg0, arg1, arg2 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLongRunningOperationState", reflect.TypeOf((*MockPrivateEndpointScope)(nil).DeleteLongRunningOperationState), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetClient", reflect.TypeOf((*MockPrivateEndpointScope)(nil).GetClient)) } // GetLongRunningOperationState mocks base method. @@ -139,25 +112,11 @@ func (mr *MockPrivateEndpointScopeMockRecorder) GetLongRunningOperationState(arg return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLongRunningOperationState", reflect.TypeOf((*MockPrivateEndpointScope)(nil).GetLongRunningOperationState), arg0, arg1, arg2) } -// HashKey mocks base method. -func (m *MockPrivateEndpointScope) HashKey() string { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "HashKey") - ret0, _ := ret[0].(string) - return ret0 -} - -// HashKey indicates an expected call of HashKey. -func (mr *MockPrivateEndpointScopeMockRecorder) HashKey() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HashKey", reflect.TypeOf((*MockPrivateEndpointScope)(nil).HashKey)) -} - // PrivateEndpointSpecs mocks base method. -func (m *MockPrivateEndpointScope) PrivateEndpointSpecs() []azure.ResourceSpecGetter { +func (m *MockPrivateEndpointScope) PrivateEndpointSpecs() []azure.ASOResourceSpecGetter[*v1api20220701.PrivateEndpoint] { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "PrivateEndpointSpecs") - ret0, _ := ret[0].([]azure.ResourceSpecGetter) + ret0, _ := ret[0].([]azure.ASOResourceSpecGetter[*v1api20220701.PrivateEndpoint]) return ret0 } @@ -179,48 +138,6 @@ func (mr *MockPrivateEndpointScopeMockRecorder) SetLongRunningOperationState(arg return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLongRunningOperationState", reflect.TypeOf((*MockPrivateEndpointScope)(nil).SetLongRunningOperationState), arg0) } -// SubscriptionID mocks base method. -func (m *MockPrivateEndpointScope) SubscriptionID() string { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SubscriptionID") - ret0, _ := ret[0].(string) - return ret0 -} - -// SubscriptionID indicates an expected call of SubscriptionID. -func (mr *MockPrivateEndpointScopeMockRecorder) SubscriptionID() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SubscriptionID", reflect.TypeOf((*MockPrivateEndpointScope)(nil).SubscriptionID)) -} - -// TenantID mocks base method. -func (m *MockPrivateEndpointScope) TenantID() string { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "TenantID") - ret0, _ := ret[0].(string) - return ret0 -} - -// TenantID indicates an expected call of TenantID. -func (mr *MockPrivateEndpointScopeMockRecorder) TenantID() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TenantID", reflect.TypeOf((*MockPrivateEndpointScope)(nil).TenantID)) -} - -// Token mocks base method. -func (m *MockPrivateEndpointScope) Token() azcore.TokenCredential { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Token") - ret0, _ := ret[0].(azcore.TokenCredential) - return ret0 -} - -// Token indicates an expected call of Token. -func (mr *MockPrivateEndpointScopeMockRecorder) Token() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Token", reflect.TypeOf((*MockPrivateEndpointScope)(nil).Token)) -} - // UpdateDeleteStatus mocks base method. func (m *MockPrivateEndpointScope) UpdateDeleteStatus(arg0 v1beta10.ConditionType, arg1 string, arg2 error) { m.ctrl.T.Helper() diff --git a/azure/services/privateendpoints/privateendpoints.go b/azure/services/privateendpoints/privateendpoints.go index 7b9345a1ba1..aab3d74db1c 100644 --- a/azure/services/privateendpoints/privateendpoints.go +++ b/azure/services/privateendpoints/privateendpoints.go @@ -17,14 +17,10 @@ limitations under the License. package privateendpoints import ( - "context" - - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4" + asonetworkv1 "github.com/Azure/azure-service-operator/v2/api/network/v1api20220701" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" - "sigs.k8s.io/cluster-api-provider-azure/azure/services/async" - "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" - "sigs.k8s.io/cluster-api-provider-azure/util/tele" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/aso" ) // ServiceName is the name of this service. @@ -32,99 +28,14 @@ const ServiceName = "privateendpoints" // PrivateEndpointScope defines the scope interface for a private endpoint. type PrivateEndpointScope interface { - azure.Authorizer - azure.AsyncStatusUpdater - PrivateEndpointSpecs() []azure.ResourceSpecGetter -} - -// Service provides operations on Azure resources. -type Service struct { - Scope PrivateEndpointScope - async.Reconciler + aso.Scope + PrivateEndpointSpecs() []azure.ASOResourceSpecGetter[*asonetworkv1.PrivateEndpoint] } // New creates a new service. -func New(scope PrivateEndpointScope) (*Service, error) { - client, err := newClient(scope) - if err != nil { - return nil, err - } - return &Service{ - Scope: scope, - Reconciler: async.New[armnetwork.PrivateEndpointsClientCreateOrUpdateResponse, - armnetwork.PrivateEndpointsClientDeleteResponse](scope, client, client), - }, nil -} - -// Name returns the service name. -func (s *Service) Name() string { - return ServiceName -} - -// Reconcile idempotently creates or updates a private endpoint. -func (s *Service) Reconcile(ctx context.Context) error { - ctx, _, done := tele.StartSpanWithLogger(ctx, "privateendpoints.Service.Reconcile") - defer done() - - ctx, cancel := context.WithTimeout(ctx, reconciler.DefaultAzureServiceReconcileTimeout) - defer cancel() - - specs := s.Scope.PrivateEndpointSpecs() - if len(specs) == 0 { - return nil - } - - // We go through the list of PrivateEndpointSpecs to reconcile each one, independently of the result of the previous one. - // If multiple errors occur, we return the most pressing one. - // Order of precedence (highest -> lowest) is: error that is not an operationNotDoneError (i.e. error creating) -> operationNotDoneError (i.e. creating in progress) -> no error (i.e. created) - var result error - for _, privateEndpointSpec := range specs { - if privateEndpointSpec == nil { - continue - } - if _, err := s.CreateOrUpdateResource(ctx, privateEndpointSpec, ServiceName); err != nil { - if !azure.IsOperationNotDoneError(err) || result == nil { - result = err - } - } - } - - s.Scope.UpdatePutStatus(infrav1.PrivateEndpointsReadyCondition, ServiceName, result) - return result -} - -// Delete deletes the private endpoint with the provided name. -func (s *Service) Delete(ctx context.Context) error { - ctx, _, done := tele.StartSpanWithLogger(ctx, "privateendpoints.Service.Delete") - defer done() - - ctx, cancel := context.WithTimeout(ctx, reconciler.DefaultAzureServiceReconcileTimeout) - defer cancel() - - specs := s.Scope.PrivateEndpointSpecs() - if len(specs) == 0 { - return nil - } - - // We go through the list of PrivateEndpointSpecs to delete each one, independently of the result of the previous one. - // If multiple errors occur, we return the most pressing one. - // Order of precedence (highest -> lowest) is: error that is not an operationNotDoneError (i.e. error deleting) -> operationNotDoneError (i.e. deleting in progress) -> no error (i.e. deleted) - var result error - for _, privateEndpointSpec := range specs { - if privateEndpointSpec == nil { - continue - } - if err := s.DeleteResource(ctx, privateEndpointSpec, ServiceName); err != nil { - if !azure.IsOperationNotDoneError(err) || result == nil { - result = err - } - } - } - s.Scope.UpdateDeleteStatus(infrav1.PrivateEndpointsReadyCondition, ServiceName, result) - return result -} - -// IsManaged returns always returns true as CAPZ does not support BYO private endpoints. -func (s *Service) IsManaged(ctx context.Context) (bool, error) { - return true, nil +func New(scope PrivateEndpointScope) *aso.Service[*asonetworkv1.PrivateEndpoint, PrivateEndpointScope] { + svc := aso.NewService[*asonetworkv1.PrivateEndpoint, PrivateEndpointScope](ServiceName, scope) + svc.ConditionType = infrav1.PrivateEndpointsReadyCondition + svc.Specs = scope.PrivateEndpointSpecs() + return svc } diff --git a/azure/services/privateendpoints/privateendpoints_test.go b/azure/services/privateendpoints/privateendpoints_test.go deleted file mode 100644 index 0509d5d6659..00000000000 --- a/azure/services/privateendpoints/privateendpoints_test.go +++ /dev/null @@ -1,258 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package privateendpoints - -import ( - "context" - "net/http" - "testing" - - "github.com/Azure/go-autorest/autorest" - . "github.com/onsi/gomega" - "go.uber.org/mock/gomock" - infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" - "sigs.k8s.io/cluster-api-provider-azure/azure" - "sigs.k8s.io/cluster-api-provider-azure/azure/services/async/mock_async" - "sigs.k8s.io/cluster-api-provider-azure/azure/services/privateendpoints/mock_privateendpoints" - gomockinternal "sigs.k8s.io/cluster-api-provider-azure/internal/test/matchers/gomock" -) - -var ( - fakePrivateEndpoint1 = PrivateEndpointSpec{ - Name: "fake-private-endpoint1", - ApplicationSecurityGroups: []string{"asg1"}, - PrivateLinkServiceConnections: []PrivateLinkServiceConnection{{PrivateLinkServiceID: "testPl", RequestMessage: "Please approve my connection."}}, - SubnetID: "mySubnet", - ResourceGroup: "my-rg", - ManualApproval: false, - } - - fakePrivateEndpoint2 = PrivateEndpointSpec{ - Name: "fake-private-endpoint2", - PrivateLinkServiceConnections: []PrivateLinkServiceConnection{{PrivateLinkServiceID: "testPl", RequestMessage: "Please approve my connection."}}, - SubnetID: "mySubnet", - ResourceGroup: "my-rg", - ManualApproval: true, - } - - fakePrivateEndpoint3 = PrivateEndpointSpec{ - Name: "fake-private-endpoint3", - ApplicationSecurityGroups: []string{"sg1"}, - PrivateLinkServiceConnections: []PrivateLinkServiceConnection{{PrivateLinkServiceID: "testPl", RequestMessage: "Please approve my connection."}}, - SubnetID: "mySubnet", - ResourceGroup: "my-rg", - ManualApproval: false, - CustomNetworkInterfaceName: "pestaticconfig", - PrivateIPAddresses: []string{"10.0.0.1"}, - } - - emptyPrivateEndpointSpec = PrivateEndpointSpec{} - fakePrivateEndpointSpecs = []azure.ResourceSpecGetter{&fakePrivateEndpoint1, &fakePrivateEndpoint2, &fakePrivateEndpoint3, &emptyPrivateEndpointSpec} - - internalError = autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: http.StatusInternalServerError}, "Internal Server Error") - notDoneError = azure.NewOperationNotDoneError(&infrav1.Future{}) -) - -func TestReconcilePrivateEndpoint(t *testing.T) { - testcases := []struct { - name string - expect func(s *mock_privateendpoints.MockPrivateEndpointScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) - expectedError string - }{ - { - name: "create a private endpoint with automatic approval", - expectedError: "", - expect: func(p *mock_privateendpoints.MockPrivateEndpointScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { - p.PrivateEndpointSpecs().Return([]azure.ResourceSpecGetter{&fakePrivateEndpoint1}) - r.CreateOrUpdateResource(gomockinternal.AContext(), &fakePrivateEndpoint1, ServiceName).Return(&fakePrivateEndpoint1, nil) - p.UpdatePutStatus(infrav1.PrivateEndpointsReadyCondition, ServiceName, nil) - }, - }, - { - name: "create a private endpoint with manual approval", - expectedError: "", - expect: func(p *mock_privateendpoints.MockPrivateEndpointScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { - p.PrivateEndpointSpecs().Return(fakePrivateEndpointSpecs[1:2]) - r.CreateOrUpdateResource(gomockinternal.AContext(), &fakePrivateEndpoint2, ServiceName).Return(&fakePrivateEndpoint2, nil) - p.UpdatePutStatus(infrav1.PrivateEndpointsReadyCondition, ServiceName, nil) - }, - }, - { - name: "create multiple private endpoints", - expectedError: "", - expect: func(p *mock_privateendpoints.MockPrivateEndpointScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { - p.PrivateEndpointSpecs().Return(fakePrivateEndpointSpecs[:2]) - r.CreateOrUpdateResource(gomockinternal.AContext(), &fakePrivateEndpoint1, ServiceName).Return(&fakePrivateEndpoint1, nil) - r.CreateOrUpdateResource(gomockinternal.AContext(), &fakePrivateEndpoint2, ServiceName).Return(&fakePrivateEndpoint2, nil) - p.UpdatePutStatus(infrav1.PrivateEndpointsReadyCondition, ServiceName, nil) - }, - }, - { - name: "return error when creating a private endpoint using an empty spec", - expectedError: internalError.Error(), - expect: func(p *mock_privateendpoints.MockPrivateEndpointScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { - p.PrivateEndpointSpecs().Return(fakePrivateEndpointSpecs[3:]) - r.CreateOrUpdateResource(gomockinternal.AContext(), &emptyPrivateEndpointSpec, ServiceName).Return(nil, internalError) - p.UpdatePutStatus(infrav1.PrivateEndpointsReadyCondition, ServiceName, internalError) - }, - }, - { - name: "not done error in creating is ignored", - expectedError: internalError.Error(), - expect: func(p *mock_privateendpoints.MockPrivateEndpointScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { - p.PrivateEndpointSpecs().Return(fakePrivateEndpointSpecs[:3]) - r.CreateOrUpdateResource(gomockinternal.AContext(), &fakePrivateEndpoint1, ServiceName).Return(&fakePrivateEndpoint1, nil) - r.CreateOrUpdateResource(gomockinternal.AContext(), &fakePrivateEndpoint2, ServiceName).Return(&fakePrivateEndpoint2, internalError) - r.CreateOrUpdateResource(gomockinternal.AContext(), &fakePrivateEndpoint3, ServiceName).Return(&fakePrivateEndpoint3, notDoneError) - p.UpdatePutStatus(infrav1.PrivateEndpointsReadyCondition, ServiceName, internalError) - }, - }, - { - name: "not done error in creating remains", - expectedError: "operation type on Azure resource / is not done", - expect: func(p *mock_privateendpoints.MockPrivateEndpointScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { - p.PrivateEndpointSpecs().Return(fakePrivateEndpointSpecs[:3]) - r.CreateOrUpdateResource(gomockinternal.AContext(), &fakePrivateEndpoint1, ServiceName).Return(&fakePrivateEndpoint1, nil) - r.CreateOrUpdateResource(gomockinternal.AContext(), &fakePrivateEndpoint2, ServiceName).Return(nil, notDoneError) - r.CreateOrUpdateResource(gomockinternal.AContext(), &fakePrivateEndpoint3, ServiceName).Return(&fakePrivateEndpoint3, nil) - p.UpdatePutStatus(infrav1.PrivateEndpointsReadyCondition, ServiceName, notDoneError) - }, - }, - } - - for _, tc := range testcases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - g := NewWithT(t) - t.Parallel() - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - scopeMock := mock_privateendpoints.NewMockPrivateEndpointScope(mockCtrl) - asyncMock := mock_async.NewMockReconciler(mockCtrl) - - tc.expect(scopeMock.EXPECT(), asyncMock.EXPECT()) - - s := &Service{ - Scope: scopeMock, - Reconciler: asyncMock, - } - - err := s.Reconcile(context.TODO()) - if tc.expectedError != "" { - g.Expect(err).To(HaveOccurred()) - g.Expect(err).To(MatchError(tc.expectedError)) - } else { - g.Expect(err).NotTo(HaveOccurred()) - } - }) - } -} - -func TestDeletePrivateEndpoints(t *testing.T) { - testcases := []struct { - name string - expect func(s *mock_privateendpoints.MockPrivateEndpointScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) - expectedError string - }{ - { - name: "delete a private endpoint", - expectedError: "", - expect: func(p *mock_privateendpoints.MockPrivateEndpointScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { - p.PrivateEndpointSpecs().Return(fakePrivateEndpointSpecs[:1]) - r.DeleteResource(gomockinternal.AContext(), &fakePrivateEndpoint1, ServiceName).Return(nil) - p.UpdateDeleteStatus(infrav1.PrivateEndpointsReadyCondition, ServiceName, nil) - }, - }, - { - name: "noop if no private endpoints specs are found", - expectedError: "", - expect: func(p *mock_privateendpoints.MockPrivateEndpointScopeMockRecorder, _ *mock_async.MockReconcilerMockRecorder) { - p.PrivateEndpointSpecs().Return([]azure.ResourceSpecGetter{}) - }, - }, - { - name: "delete multiple private endpoints", - expectedError: "", - expect: func(p *mock_privateendpoints.MockPrivateEndpointScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { - p.PrivateEndpointSpecs().Return(fakePrivateEndpointSpecs[:2]) - r.DeleteResource(gomockinternal.AContext(), &fakePrivateEndpoint1, ServiceName).Return(nil) - r.DeleteResource(gomockinternal.AContext(), &fakePrivateEndpoint2, ServiceName).Return(nil) - p.UpdateDeleteStatus(infrav1.PrivateEndpointsReadyCondition, ServiceName, nil) - }, - }, - { - name: "error in deleting peering", - expectedError: internalError.Error(), - expect: func(p *mock_privateendpoints.MockPrivateEndpointScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { - p.PrivateEndpointSpecs().Return(fakePrivateEndpointSpecs[:2]) - r.DeleteResource(gomockinternal.AContext(), &fakePrivateEndpoint1, ServiceName).Return(nil) - r.DeleteResource(gomockinternal.AContext(), &fakePrivateEndpoint2, ServiceName).Return(internalError) - p.UpdateDeleteStatus(infrav1.PrivateEndpointsReadyCondition, ServiceName, internalError) - }, - }, - { - name: "not done error in deleting is ignored", - expectedError: internalError.Error(), - expect: func(p *mock_privateendpoints.MockPrivateEndpointScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { - p.PrivateEndpointSpecs().Return(fakePrivateEndpointSpecs[:3]) - r.DeleteResource(gomockinternal.AContext(), &fakePrivateEndpoint1, ServiceName).Return(nil) - r.DeleteResource(gomockinternal.AContext(), &fakePrivateEndpoint2, ServiceName).Return(internalError) - r.DeleteResource(gomockinternal.AContext(), &fakePrivateEndpoint3, ServiceName).Return(notDoneError) - p.UpdateDeleteStatus(infrav1.PrivateEndpointsReadyCondition, ServiceName, internalError) - }, - }, - { - name: "not done error in deleting remains", - expectedError: "operation type on Azure resource / is not done", - expect: func(p *mock_privateendpoints.MockPrivateEndpointScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { - p.PrivateEndpointSpecs().Return(fakePrivateEndpointSpecs[:3]) - r.DeleteResource(gomockinternal.AContext(), &fakePrivateEndpoint1, ServiceName).Return(nil) - r.DeleteResource(gomockinternal.AContext(), &fakePrivateEndpoint2, ServiceName).Return(nil) - r.DeleteResource(gomockinternal.AContext(), &fakePrivateEndpoint3, ServiceName).Return(notDoneError) - p.UpdateDeleteStatus(infrav1.PrivateEndpointsReadyCondition, ServiceName, notDoneError) - }, - }, - } - - for _, tc := range testcases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - g := NewWithT(t) - - t.Parallel() - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - scopeMock := mock_privateendpoints.NewMockPrivateEndpointScope(mockCtrl) - asyncMock := mock_async.NewMockReconciler(mockCtrl) - - tc.expect(scopeMock.EXPECT(), asyncMock.EXPECT()) - - s := &Service{ - Scope: scopeMock, - Reconciler: asyncMock, - } - - err := s.Delete(context.TODO()) - if tc.expectedError != "" { - g.Expect(err).To(HaveOccurred()) - g.Expect(err).To(MatchError(tc.expectedError)) - } else { - g.Expect(err).NotTo(HaveOccurred()) - } - }) - } -} diff --git a/azure/services/privateendpoints/spec.go b/azure/services/privateendpoints/spec.go index bc7dd558de3..f2ab5f0c2ae 100644 --- a/azure/services/privateendpoints/spec.go +++ b/azure/services/privateendpoints/spec.go @@ -19,16 +19,12 @@ package privateendpoints import ( "context" "sort" - "time" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4" - "github.com/google/go-cmp/cmp" - "github.com/pkg/errors" + asonetworkv1 "github.com/Azure/azure-service-operator/v2/api/network/v1api20220701" + "github.com/Azure/azure-service-operator/v2/pkg/genruntime" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" - "sigs.k8s.io/cluster-api-provider-azure/azure" - "sigs.k8s.io/cluster-api-provider-azure/azure/converters" - "sigs.k8s.io/cluster-api-provider-azure/util/tele" ) // PrivateLinkServiceConnection defines the specification for a private link service connection associated with a private endpoint. @@ -42,6 +38,7 @@ type PrivateLinkServiceConnection struct { // PrivateEndpointSpec defines the specification for a private endpoint. type PrivateEndpointSpec struct { Name string + Namespace string ResourceGroup string Location string CustomNetworkInterfaceName string @@ -54,209 +51,107 @@ type PrivateEndpointSpec struct { ClusterName string } -// ResourceName returns the name of the private endpoint. -func (s *PrivateEndpointSpec) ResourceName() string { - return s.Name -} - -// ResourceGroupName returns the name of the resource group. -func (s *PrivateEndpointSpec) ResourceGroupName() string { - return s.ResourceGroup -} - -// OwnerResourceName is a no-op for private endpoints. -func (s *PrivateEndpointSpec) OwnerResourceName() string { - return "" -} - -// Parameters returns the parameters for the PrivateEndpointSpec. -func (s *PrivateEndpointSpec) Parameters(ctx context.Context, existing interface{}) (interface{}, error) { - _, log, done := tele.StartSpanWithLogger(ctx, "privateendpoints.Service.Parameters") - defer done() - - privateEndpointProperties := armnetwork.PrivateEndpointProperties{ - Subnet: &armnetwork.Subnet{ - ID: &s.SubnetID, - Properties: &armnetwork.SubnetPropertiesFormat{ - PrivateEndpointNetworkPolicies: ptr.To(armnetwork.VirtualNetworkPrivateEndpointNetworkPoliciesDisabled), - PrivateLinkServiceNetworkPolicies: ptr.To(armnetwork.VirtualNetworkPrivateLinkServiceNetworkPoliciesEnabled), - }, +// ResourceRef implements azure.ASOResourceSpecGetter. +func (s *PrivateEndpointSpec) ResourceRef() *asonetworkv1.PrivateEndpoint { + return &asonetworkv1.PrivateEndpoint{ + ObjectMeta: metav1.ObjectMeta{ + Name: s.Name, + Namespace: s.Namespace, }, } +} - privateEndpointProperties.CustomNetworkInterfaceName = ptr.To(s.CustomNetworkInterfaceName) - - privateIPAddresses := make([]*armnetwork.PrivateEndpointIPConfiguration, 0, len(s.PrivateIPAddresses)) - for _, address := range s.PrivateIPAddresses { - ipConfig := &armnetwork.PrivateEndpointIPConfigurationProperties{PrivateIPAddress: ptr.To(address)} - - privateIPAddresses = append(privateIPAddresses, &armnetwork.PrivateEndpointIPConfiguration{ - Properties: ipConfig, - }) +// Parameters implements azure.ASOResourceSpecGetter. +func (s *PrivateEndpointSpec) Parameters(ctx context.Context, existingPrivateEndpoint *asonetworkv1.PrivateEndpoint) (*asonetworkv1.PrivateEndpoint, error) { + privateEndpoint := &asonetworkv1.PrivateEndpoint{} + if existingPrivateEndpoint != nil { + privateEndpoint = existingPrivateEndpoint } - privateEndpointProperties.IPConfigurations = privateIPAddresses - - privateLinkServiceConnections := make([]*armnetwork.PrivateLinkServiceConnection, 0, len(s.PrivateLinkServiceConnections)) - for _, privateLinkServiceConnection := range s.PrivateLinkServiceConnections { - linkServiceConnection := &armnetwork.PrivateLinkServiceConnection{ - Name: ptr.To(privateLinkServiceConnection.Name), - Properties: &armnetwork.PrivateLinkServiceConnectionProperties{ - PrivateLinkServiceID: ptr.To(privateLinkServiceConnection.PrivateLinkServiceID), - }, - } - if len(privateLinkServiceConnection.GroupIDs) > 0 { - linkServiceConnection.Properties.GroupIDs = azure.PtrSlice(&privateLinkServiceConnection.GroupIDs) - } - - if privateLinkServiceConnection.RequestMessage != "" { - linkServiceConnection.Properties.RequestMessage = ptr.To(privateLinkServiceConnection.RequestMessage) + if len(s.ApplicationSecurityGroups) > 0 { + applicationSecurityGroups := make([]asonetworkv1.ApplicationSecurityGroupSpec_PrivateEndpoint_SubResourceEmbedded, 0, len(s.ApplicationSecurityGroups)) + for _, applicationSecurityGroup := range s.ApplicationSecurityGroups { + applicationSecurityGroups = append(applicationSecurityGroups, asonetworkv1.ApplicationSecurityGroupSpec_PrivateEndpoint_SubResourceEmbedded{ + Reference: &genruntime.ResourceReference{ + ARMID: applicationSecurityGroup, + }, + }) } - privateLinkServiceConnections = append(privateLinkServiceConnections, linkServiceConnection) - } - - if s.ManualApproval { - privateEndpointProperties.ManualPrivateLinkServiceConnections = privateLinkServiceConnections - privateEndpointProperties.PrivateLinkServiceConnections = []*armnetwork.PrivateLinkServiceConnection{} - } else { - privateEndpointProperties.PrivateLinkServiceConnections = privateLinkServiceConnections - privateEndpointProperties.ManualPrivateLinkServiceConnections = []*armnetwork.PrivateLinkServiceConnection{} - } - - applicationSecurityGroups := make([]armnetwork.ApplicationSecurityGroup, 0, len(s.ApplicationSecurityGroups)) - - for _, applicationSecurityGroup := range s.ApplicationSecurityGroups { - applicationSecurityGroups = append(applicationSecurityGroups, armnetwork.ApplicationSecurityGroup{ - ID: ptr.To(applicationSecurityGroup), + // Sort the slices in order to get the same order of elements for both new and existing application Security Groups. + sort.SliceStable(applicationSecurityGroups, func(i, j int) bool { + return applicationSecurityGroups[i].Reference.ARMID < applicationSecurityGroups[j].Reference.ARMID }) + privateEndpoint.Spec.ApplicationSecurityGroups = applicationSecurityGroups } - privateEndpointProperties.ApplicationSecurityGroups = azure.PtrSlice(&applicationSecurityGroups) + privateEndpoint.Spec.AzureName = s.Name + privateEndpoint.Spec.CustomNetworkInterfaceName = ptr.To(s.CustomNetworkInterfaceName) + privateEndpoint.Spec.Location = ptr.To(s.Location) - newPrivateEndpoint := armnetwork.PrivateEndpoint{ - Name: ptr.To(s.Name), - Properties: &privateEndpointProperties, - Tags: converters.TagsToMap(infrav1.Build(infrav1.BuildParams{ - ClusterName: s.ClusterName, - Lifecycle: infrav1.ResourceLifecycleOwned, - Name: ptr.To(s.Name), - Additional: s.AdditionalTags, - })), + if len(s.PrivateIPAddresses) > 0 { + privateIPAddresses := make([]asonetworkv1.PrivateEndpointIPConfiguration, 0, len(s.PrivateIPAddresses)) + for _, address := range s.PrivateIPAddresses { + ipConfig := asonetworkv1.PrivateEndpointIPConfiguration{PrivateIPAddress: ptr.To(address)} + privateIPAddresses = append(privateIPAddresses, ipConfig) + } + sort.SliceStable(privateIPAddresses, func(i, j int) bool { + return *privateIPAddresses[i].PrivateIPAddress < *privateIPAddresses[j].PrivateIPAddress + }) + privateEndpoint.Spec.IpConfigurations = privateIPAddresses } - if s.Location != "" { - newPrivateEndpoint.Location = ptr.To(s.Location) - } + if len(s.PrivateLinkServiceConnections) > 0 { + privateLinkServiceConnections := make([]asonetworkv1.PrivateLinkServiceConnection, 0, len(s.PrivateLinkServiceConnections)) + for _, privateLinkServiceConnection := range s.PrivateLinkServiceConnections { + linkServiceConnection := asonetworkv1.PrivateLinkServiceConnection{ + Name: ptr.To(privateLinkServiceConnection.Name), + PrivateLinkServiceReference: &genruntime.ResourceReference{ + ARMID: privateLinkServiceConnection.PrivateLinkServiceID, + }, + } - if existing != nil { - existingPE, ok := existing.(armnetwork.PrivateEndpoint) - if !ok { - return nil, errors.Errorf("%T is not a network.PrivateEndpoint", existing) - } + if len(privateLinkServiceConnection.GroupIDs) > 0 { + linkServiceConnection.GroupIds = privateLinkServiceConnection.GroupIDs + } - ps := ptr.Deref(existingPE.Properties.ProvisioningState, "") - if string(ps) != string(infrav1.Canceled) && string(ps) != string(infrav1.Failed) && string(ps) != string(infrav1.Succeeded) { - return nil, azure.WithTransientError(errors.Errorf("Unable to update existing private endpoint in non-terminal state. Service Endpoint must be in one of the following provisioning states: Canceled, Failed, or Succeeded. Actual state: %s", ps), 20*time.Second) + if privateLinkServiceConnection.RequestMessage != "" { + linkServiceConnection.RequestMessage = ptr.To(privateLinkServiceConnection.RequestMessage) + } + privateLinkServiceConnections = append(privateLinkServiceConnections, linkServiceConnection) } + sort.SliceStable(privateLinkServiceConnections, func(i, j int) bool { + return *privateLinkServiceConnections[i].Name < *privateLinkServiceConnections[j].Name + }) - normalizedExistingPE := normalizePrivateEndpoint(existingPE, newPrivateEndpoint) - normalizedExistingPE = sortSlicesPrivateEndpoint(normalizedExistingPE) - - newPrivateEndpoint = sortSlicesPrivateEndpoint(newPrivateEndpoint) - - diff := cmp.Diff(&normalizedExistingPE, &newPrivateEndpoint) - if diff == "" { - // PrivateEndpoint is up-to-date, nothing to do - log.V(4).Info("no changes found between user-updated spec and existing spec") - return nil, nil + if s.ManualApproval { + privateEndpoint.Spec.ManualPrivateLinkServiceConnections = privateLinkServiceConnections + } else { + privateEndpoint.Spec.PrivateLinkServiceConnections = privateLinkServiceConnections } - log.V(4).Info("found a diff between the desired spec and the existing privateendpoint", "difference", diff) } - return newPrivateEndpoint, nil -} - -func normalizePrivateEndpoint(existingPE, newPrivateEndpoint armnetwork.PrivateEndpoint) armnetwork.PrivateEndpoint { - normalizedExistingPE := armnetwork.PrivateEndpoint{ - Name: existingPE.Name, - Location: existingPE.Location, - Properties: &armnetwork.PrivateEndpointProperties{ - Subnet: &armnetwork.Subnet{ - ID: existingPE.Properties.Subnet.ID, - Properties: &armnetwork.SubnetPropertiesFormat{ - PrivateEndpointNetworkPolicies: newPrivateEndpoint.Properties.Subnet.Properties.PrivateEndpointNetworkPolicies, - PrivateLinkServiceNetworkPolicies: newPrivateEndpoint.Properties.Subnet.Properties.PrivateLinkServiceNetworkPolicies, - }, - }, - ApplicationSecurityGroups: existingPE.Properties.ApplicationSecurityGroups, - IPConfigurations: existingPE.Properties.IPConfigurations, - CustomNetworkInterfaceName: existingPE.Properties.CustomNetworkInterfaceName, - }, - Tags: existingPE.Tags, - } - if existingPE.Properties != nil && existingPE.Properties.Subnet != nil && existingPE.Properties.Subnet.Properties != nil { - normalizedExistingPE.Properties.Subnet.Properties.PrivateEndpointNetworkPolicies = existingPE.Properties.Subnet.Properties.PrivateEndpointNetworkPolicies - normalizedExistingPE.Properties.Subnet.Properties.PrivateLinkServiceNetworkPolicies = existingPE.Properties.Subnet.Properties.PrivateLinkServiceNetworkPolicies + privateEndpoint.Spec.Owner = &genruntime.KnownResourceReference{ + Name: s.ResourceGroup, } - existingPrivateLinkServiceConnections := make([]*armnetwork.PrivateLinkServiceConnection, 0, len(existingPE.Properties.PrivateLinkServiceConnections)) - for _, privateLinkServiceConnection := range existingPE.Properties.PrivateLinkServiceConnections { - existingPrivateLinkServiceConnections = append(existingPrivateLinkServiceConnections, &armnetwork.PrivateLinkServiceConnection{ - Name: privateLinkServiceConnection.Name, - Properties: &armnetwork.PrivateLinkServiceConnectionProperties{ - PrivateLinkServiceID: privateLinkServiceConnection.Properties.PrivateLinkServiceID, - RequestMessage: privateLinkServiceConnection.Properties.RequestMessage, - GroupIDs: privateLinkServiceConnection.Properties.GroupIDs, - }, - }) + privateEndpoint.Spec.Subnet = &asonetworkv1.Subnet_PrivateEndpoint_SubResourceEmbedded{ + Reference: &genruntime.ResourceReference{ + ARMID: s.SubnetID, + }, } - normalizedExistingPE.Properties.PrivateLinkServiceConnections = existingPrivateLinkServiceConnections - existingManualPrivateLinkServiceConnections := make([]*armnetwork.PrivateLinkServiceConnection, 0, len(existingPE.Properties.ManualPrivateLinkServiceConnections)) - for _, manualPrivateLinkServiceConnection := range existingPE.Properties.ManualPrivateLinkServiceConnections { - existingManualPrivateLinkServiceConnections = append(existingManualPrivateLinkServiceConnections, &armnetwork.PrivateLinkServiceConnection{ - Name: manualPrivateLinkServiceConnection.Name, - Properties: &armnetwork.PrivateLinkServiceConnectionProperties{ - PrivateLinkServiceID: manualPrivateLinkServiceConnection.Properties.PrivateLinkServiceID, - RequestMessage: manualPrivateLinkServiceConnection.Properties.RequestMessage, - GroupIDs: manualPrivateLinkServiceConnection.Properties.GroupIDs, - }, - }) - } - normalizedExistingPE.Properties.ManualPrivateLinkServiceConnections = existingManualPrivateLinkServiceConnections + privateEndpoint.Spec.Tags = infrav1.Build(infrav1.BuildParams{ + ClusterName: s.ClusterName, + Lifecycle: infrav1.ResourceLifecycleOwned, + Name: ptr.To(s.Name), + Additional: s.AdditionalTags, + }) - return normalizedExistingPE + return privateEndpoint, nil } -// Sort all slices in order to get the same order of elements for both new and existing private endpoints. -func sortSlicesPrivateEndpoint(privateEndpoint armnetwork.PrivateEndpoint) armnetwork.PrivateEndpoint { - // Sort ManualPrivateLinkServiceConnections - if privateEndpoint.Properties.ManualPrivateLinkServiceConnections != nil { - sort.SliceStable(privateEndpoint.Properties.ManualPrivateLinkServiceConnections, func(i, j int) bool { - return *privateEndpoint.Properties.ManualPrivateLinkServiceConnections[i].Name < *privateEndpoint.Properties.ManualPrivateLinkServiceConnections[j].Name - }) - } - - // Sort PrivateLinkServiceConnections - if privateEndpoint.Properties.PrivateLinkServiceConnections != nil { - sort.SliceStable(privateEndpoint.Properties.PrivateLinkServiceConnections, func(i, j int) bool { - return *privateEndpoint.Properties.PrivateLinkServiceConnections[i].Name < *privateEndpoint.Properties.PrivateLinkServiceConnections[j].Name - }) - } - - // Sort IPConfigurations - if privateEndpoint.Properties.IPConfigurations != nil { - sort.SliceStable(privateEndpoint.Properties.IPConfigurations, func(i, j int) bool { - return *privateEndpoint.Properties.IPConfigurations[i].Properties.PrivateIPAddress < *privateEndpoint.Properties.IPConfigurations[j].Properties.PrivateIPAddress - }) - } - - // Sort ApplicationSecurityGroups - if privateEndpoint.Properties.ApplicationSecurityGroups != nil { - sort.SliceStable(privateEndpoint.Properties.ApplicationSecurityGroups, func(i, j int) bool { - return *privateEndpoint.Properties.ApplicationSecurityGroups[i].Name < *privateEndpoint.Properties.ApplicationSecurityGroups[j].Name - }) - } - - return privateEndpoint +// WasManaged implements azure.ASOResourceSpecGetter. +// It always returns true since CAPZ doesn't support BYO private endpoints. +func (s *PrivateEndpointSpec) WasManaged(privateEndpoint *asonetworkv1.PrivateEndpoint) bool { + return true } diff --git a/azure/services/privateendpoints/spec_test.go b/azure/services/privateendpoints/spec_test.go index f6aa34db0f2..c2c4552d0c9 100644 --- a/azure/services/privateendpoints/spec_test.go +++ b/azure/services/privateendpoints/spec_test.go @@ -20,332 +20,205 @@ import ( "context" "testing" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4" + asonetworkv1 "github.com/Azure/azure-service-operator/v2/api/network/v1api20220701" + "github.com/Azure/azure-service-operator/v2/pkg/genruntime" . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" ) var ( - privateEndpoint1 = infrav1.PrivateEndpointSpec{ - Name: "test-private-endpoint1", - ApplicationSecurityGroups: []string{"asg1"}, - PrivateLinkServiceConnections: []infrav1.PrivateLinkServiceConnection{{PrivateLinkServiceID: "testPl", RequestMessage: "Please approve my connection."}}, - ManualApproval: false, + fakePrivateEndpoint = PrivateEndpointSpec{ + Name: "test_private_endpoint_1", + Namespace: "test_ns", + ResourceGroup: "test_rg", + Location: "test_location", + CustomNetworkInterfaceName: "test_if_name", + PrivateIPAddresses: []string{"1.2.3.4", "5.6.7.8"}, + SubnetID: "test_subnet_id", + ApplicationSecurityGroups: []string{"test_asg1", "test_asg2"}, + ManualApproval: false, + PrivateLinkServiceConnections: []PrivateLinkServiceConnection{ + { + Name: "test_plsc_1", + PrivateLinkServiceID: "test_pl", + RequestMessage: "Please approve my connection.", + GroupIDs: []string{"aa", "bb"}}, + }, + AdditionalTags: infrav1.Tags{"test_tag1": "test_value1", "test_tag2": "test_value2"}, + ClusterName: "test_cluster", + } + + fakeExtendedLocation = asonetworkv1.ExtendedLocation{ + Name: ptr.To("extended_location_name"), + Type: ptr.To(asonetworkv1.ExtendedLocationType_EdgeZone), } - privateEndpoint1Manual = infrav1.PrivateEndpointSpec{ - Name: "test-private-endpoint-manual", - ApplicationSecurityGroups: []string{"asg1"}, - PrivateLinkServiceConnections: []infrav1.PrivateLinkServiceConnection{{PrivateLinkServiceID: "testPl", RequestMessage: "Please approve my connection.", GroupIDs: []string{"aa", "bb"}}}, - ManualApproval: true, - CustomNetworkInterfaceName: "test-if-name", - PrivateIPAddresses: []string{"10.0.0.1", "10.0.0.2"}, - Location: "test-location", + fakeASOPrivateEndpoint = asonetworkv1.PrivateEndpoint{ + ObjectMeta: metav1.ObjectMeta{ + Name: fakePrivateEndpoint.Name, + Namespace: fakePrivateEndpoint.Namespace, + }, + Spec: asonetworkv1.PrivateEndpoint_Spec{ + ApplicationSecurityGroups: []asonetworkv1.ApplicationSecurityGroupSpec_PrivateEndpoint_SubResourceEmbedded{ + { + Reference: &genruntime.ResourceReference{ + ARMID: fakePrivateEndpoint.ApplicationSecurityGroups[0], + }, + }, + { + Reference: &genruntime.ResourceReference{ + ARMID: fakePrivateEndpoint.ApplicationSecurityGroups[1], + }, + }, + }, + AzureName: fakePrivateEndpoint.Name, + PrivateLinkServiceConnections: []asonetworkv1.PrivateLinkServiceConnection{{ + Name: ptr.To(fakePrivateEndpoint.PrivateLinkServiceConnections[0].Name), + PrivateLinkServiceReference: &genruntime.ResourceReference{ + ARMID: fakePrivateEndpoint.PrivateLinkServiceConnections[0].PrivateLinkServiceID, + }, + GroupIds: fakePrivateEndpoint.PrivateLinkServiceConnections[0].GroupIDs, + RequestMessage: ptr.To(fakePrivateEndpoint.PrivateLinkServiceConnections[0].RequestMessage), + }}, + Owner: &genruntime.KnownResourceReference{ + Name: fakePrivateEndpoint.ResourceGroup, + }, + Subnet: &asonetworkv1.Subnet_PrivateEndpoint_SubResourceEmbedded{ + Reference: &genruntime.ResourceReference{ + ARMID: fakePrivateEndpoint.SubnetID, + }, + }, + IpConfigurations: []asonetworkv1.PrivateEndpointIPConfiguration{ + { + PrivateIPAddress: ptr.To(fakePrivateEndpoint.PrivateIPAddresses[0]), + }, + { + PrivateIPAddress: ptr.To(fakePrivateEndpoint.PrivateIPAddresses[1]), + }, + }, + CustomNetworkInterfaceName: ptr.To(fakePrivateEndpoint.CustomNetworkInterfaceName), + Location: ptr.To(fakePrivateEndpoint.Location), + Tags: map[string]string{"sigs.k8s.io_cluster-api-provider-azure_cluster_test_cluster": "owned", "Name": "test_private_endpoint_1", "test_tag1": "test_value1", "test_tag2": "test_value2"}, + }, } - privateEndpoint2 = infrav1.PrivateEndpointSpec{ - Name: "test-private-endpoint2", - ApplicationSecurityGroups: []string{"asg1"}, - PrivateLinkServiceConnections: []infrav1.PrivateLinkServiceConnection{{PrivateLinkServiceID: "testPl", RequestMessage: "Please approve my connection.", GroupIDs: []string{"aa", "bb"}}}, - ManualApproval: false, - CustomNetworkInterfaceName: "test-if-name", - PrivateIPAddresses: []string{"10.0.0.1", "10.0.0.2"}, - Location: "test-location", + fakeASOPrivateEndpointsStatus = asonetworkv1.PrivateEndpoint_STATUS_PrivateEndpoint_SubResourceEmbedded{ + ApplicationSecurityGroups: []asonetworkv1.ApplicationSecurityGroup_STATUS_PrivateEndpoint_SubResourceEmbedded{ + { + Id: ptr.To(fakePrivateEndpoint.ApplicationSecurityGroups[0]), + }, + { + Id: ptr.To(fakePrivateEndpoint.ApplicationSecurityGroups[1]), + }, + }, + Name: ptr.To(fakePrivateEndpoint.Name), + // ... other fields truncated for brevity } ) +func getASOPrivateEndpoint(changes ...func(*asonetworkv1.PrivateEndpoint)) *asonetworkv1.PrivateEndpoint { + privateEndpoint := fakeASOPrivateEndpoint.DeepCopy() + for _, change := range changes { + change(privateEndpoint) + } + return privateEndpoint +} + func TestParameters(t *testing.T) { testcases := []struct { name string spec *PrivateEndpointSpec - existing interface{} - expect func(g *WithT, result interface{}) + existing *asonetworkv1.PrivateEndpoint + expect func(g *WithT, result asonetworkv1.PrivateEndpoint) expectedError string }{ { - name: "PrivateEndpoint already exists with the same config", - spec: &PrivateEndpointSpec{ - Name: privateEndpoint1.Name, - ResourceGroup: "test-group", - ClusterName: "my-cluster", - ApplicationSecurityGroups: privateEndpoint1.ApplicationSecurityGroups, - PrivateLinkServiceConnections: []PrivateLinkServiceConnection{{ - Name: privateEndpoint1.PrivateLinkServiceConnections[0].Name, - GroupIDs: privateEndpoint1.PrivateLinkServiceConnections[0].GroupIDs, - PrivateLinkServiceID: privateEndpoint1.PrivateLinkServiceConnections[0].PrivateLinkServiceID, - RequestMessage: privateEndpoint1.PrivateLinkServiceConnections[0].RequestMessage, - }}, - SubnetID: "test-subnet", - }, - // See https://learn.microsoft.com/en-us/rest/api/virtualnetwork/private-endpoints/get?tabs=Go for more options - existing: armnetwork.PrivateEndpoint{ - Name: ptr.To("test-private-endpoint1"), - Properties: &armnetwork.PrivateEndpointProperties{ - Subnet: &armnetwork.Subnet{ - ID: ptr.To("test-subnet"), - }, - ApplicationSecurityGroups: []*armnetwork.ApplicationSecurityGroup{{ - ID: ptr.To("asg1"), - }}, - CustomNetworkInterfaceName: ptr.To(""), - IPConfigurations: []*armnetwork.PrivateEndpointIPConfiguration{}, - PrivateLinkServiceConnections: []*armnetwork.PrivateLinkServiceConnection{{ - Name: ptr.To(privateEndpoint1.PrivateLinkServiceConnections[0].Name), - Properties: &armnetwork.PrivateLinkServiceConnectionProperties{ - PrivateLinkServiceID: ptr.To(privateEndpoint1.PrivateLinkServiceConnections[0].PrivateLinkServiceID), - GroupIDs: nil, - RequestMessage: ptr.To(privateEndpoint1.PrivateLinkServiceConnections[0].RequestMessage), - }, - }}, - ManualPrivateLinkServiceConnections: []*armnetwork.PrivateLinkServiceConnection{}, - ProvisioningState: ptr.To(armnetwork.ProvisioningStateSucceeded), - }, - Tags: map[string]*string{"sigs.k8s.io_cluster-api-provider-azure_cluster_my-cluster": ptr.To("owned"), "Name": ptr.To("test-private-endpoint1")}, - }, - expect: func(g *WithT, result interface{}) { - g.Expect(result).To(BeNil()) + name: "Creating a new PrivateEndpoint", + spec: ptr.To(fakePrivateEndpoint), + existing: nil, + expect: func(g *WithT, result asonetworkv1.PrivateEndpoint) { + g.Expect(result).To(Not(BeNil())) + + // ObjectMeta is populated later in the codeflow + g.Expect(result.ObjectMeta).To(Equal(metav1.ObjectMeta{})) + + // Spec is populated from the spec passed in + g.Expect(result.Spec).To(Equal(getASOPrivateEndpoint().Spec)) }, }, { - name: "PrivateEndpoint without AppplicationSecurityGroups already exists with the same config", - spec: &PrivateEndpointSpec{ - Name: privateEndpoint1.Name, - ResourceGroup: "test-group", - ClusterName: "my-cluster", - ApplicationSecurityGroups: nil, - PrivateLinkServiceConnections: []PrivateLinkServiceConnection{{ - Name: privateEndpoint1.PrivateLinkServiceConnections[0].Name, - GroupIDs: privateEndpoint1.PrivateLinkServiceConnections[0].GroupIDs, - PrivateLinkServiceID: privateEndpoint1.PrivateLinkServiceConnections[0].PrivateLinkServiceID, - RequestMessage: privateEndpoint1.PrivateLinkServiceConnections[0].RequestMessage, - }}, - SubnetID: "test-subnet", - }, - // See https://learn.microsoft.com/en-us/rest/api/virtualnetwork/private-endpoints/get?tabs=Go for more options - existing: armnetwork.PrivateEndpoint{ - Name: ptr.To("test-private-endpoint1"), - Properties: &armnetwork.PrivateEndpointProperties{ - Subnet: &armnetwork.Subnet{ - ID: ptr.To("test-subnet"), - }, - ApplicationSecurityGroups: nil, - CustomNetworkInterfaceName: ptr.To(""), - IPConfigurations: []*armnetwork.PrivateEndpointIPConfiguration{}, - PrivateLinkServiceConnections: []*armnetwork.PrivateLinkServiceConnection{{ - Name: ptr.To(privateEndpoint1.PrivateLinkServiceConnections[0].Name), - Properties: &armnetwork.PrivateLinkServiceConnectionProperties{ - PrivateLinkServiceID: ptr.To(privateEndpoint1.PrivateLinkServiceConnections[0].PrivateLinkServiceID), - GroupIDs: nil, - RequestMessage: ptr.To(privateEndpoint1.PrivateLinkServiceConnections[0].RequestMessage), - }, - }}, - ManualPrivateLinkServiceConnections: []*armnetwork.PrivateLinkServiceConnection{}, - ProvisioningState: ptr.To(armnetwork.ProvisioningStateSucceeded), + name: "user updates to private endpoints Extended Location should be accepted", + spec: ptr.To(fakePrivateEndpoint), + existing: getASOPrivateEndpoint( + // user added ExtendedLocation + func(endpoint *asonetworkv1.PrivateEndpoint) { + endpoint.Spec.ExtendedLocation = fakeExtendedLocation.DeepCopy() }, - Tags: map[string]*string{"sigs.k8s.io_cluster-api-provider-azure_cluster_my-cluster": ptr.To("owned"), "Name": ptr.To("test-private-endpoint1")}, - }, - expect: func(g *WithT, result interface{}) { - g.Expect(result).To(BeNil()) - }, - }, - { - name: "PrivateEndpoint with manual approval already exists with the same config", - spec: &PrivateEndpointSpec{ - Name: privateEndpoint1Manual.Name, - ResourceGroup: "test-group", - ClusterName: "my-cluster", - ApplicationSecurityGroups: privateEndpoint1Manual.ApplicationSecurityGroups, - PrivateLinkServiceConnections: []PrivateLinkServiceConnection{{ - Name: privateEndpoint1Manual.PrivateLinkServiceConnections[0].Name, - GroupIDs: privateEndpoint1Manual.PrivateLinkServiceConnections[0].GroupIDs, - PrivateLinkServiceID: privateEndpoint1Manual.PrivateLinkServiceConnections[0].PrivateLinkServiceID, - RequestMessage: privateEndpoint1Manual.PrivateLinkServiceConnections[0].RequestMessage, - }}, - SubnetID: "test-subnet", - ManualApproval: privateEndpoint1Manual.ManualApproval, - }, - // See https://learn.microsoft.com/en-us/rest/api/virtualnetwork/private-endpoints/get?tabs=Go for more options - existing: armnetwork.PrivateEndpoint{ - Name: ptr.To("test-private-endpoint-manual"), - Properties: &armnetwork.PrivateEndpointProperties{ - Subnet: &armnetwork.Subnet{ - ID: ptr.To("test-subnet"), - }, - ApplicationSecurityGroups: []*armnetwork.ApplicationSecurityGroup{{ - ID: ptr.To("asg1"), - }}, - CustomNetworkInterfaceName: ptr.To(""), - IPConfigurations: []*armnetwork.PrivateEndpointIPConfiguration{}, - ManualPrivateLinkServiceConnections: []*armnetwork.PrivateLinkServiceConnection{{ - Name: ptr.To(privateEndpoint1Manual.PrivateLinkServiceConnections[0].Name), - Properties: &armnetwork.PrivateLinkServiceConnectionProperties{ - PrivateLinkServiceID: ptr.To(privateEndpoint1Manual.PrivateLinkServiceConnections[0].PrivateLinkServiceID), - GroupIDs: []*string{ptr.To("aa"), ptr.To("bb")}, - RequestMessage: ptr.To(privateEndpoint1Manual.PrivateLinkServiceConnections[0].RequestMessage), - }, - }}, - PrivateLinkServiceConnections: []*armnetwork.PrivateLinkServiceConnection{}, - ProvisioningState: ptr.To(armnetwork.ProvisioningStateSucceeded), + // user added Status + func(endpoint *asonetworkv1.PrivateEndpoint) { + endpoint.Status = fakeASOPrivateEndpointsStatus }, - Tags: map[string]*string{"sigs.k8s.io_cluster-api-provider-azure_cluster_my-cluster": ptr.To("owned"), "Name": ptr.To("test-private-endpoint-manual")}, - }, - expect: func(g *WithT, result interface{}) { - g.Expect(result).To(BeNil()) + ), + expect: func(g *WithT, result asonetworkv1.PrivateEndpoint) { + g.Expect(result).To(Not(BeNil())) + resultantASOPrivateEndpoint := getASOPrivateEndpoint( + func(endpoint *asonetworkv1.PrivateEndpoint) { + endpoint.Spec.ExtendedLocation = fakeExtendedLocation.DeepCopy() + }, + ) + + // ObjectMeta should be carried over from existing private endpoint. + g.Expect(result.ObjectMeta).To(Equal(resultantASOPrivateEndpoint.ObjectMeta)) + + // Extended location addition is accepted. + g.Expect(result.Spec).To(Equal(resultantASOPrivateEndpoint.Spec)) + + // Status should be carried over. + g.Expect(result.Status).To(Equal(fakeASOPrivateEndpointsStatus)) }, }, { - name: "PrivateEndpoint already exists, but missing an IP address", - spec: &PrivateEndpointSpec{ - Name: privateEndpoint2.Name, - Location: privateEndpoint2.Location, - ResourceGroup: "test-group", - ClusterName: "my-cluster", - ApplicationSecurityGroups: privateEndpoint2.ApplicationSecurityGroups, - PrivateLinkServiceConnections: []PrivateLinkServiceConnection{{ - Name: privateEndpoint2.PrivateLinkServiceConnections[0].Name, - GroupIDs: privateEndpoint2.PrivateLinkServiceConnections[0].GroupIDs, - PrivateLinkServiceID: privateEndpoint2.PrivateLinkServiceConnections[0].PrivateLinkServiceID, - RequestMessage: privateEndpoint2.PrivateLinkServiceConnections[0].RequestMessage, - }}, - SubnetID: "test-subnet", - PrivateIPAddresses: privateEndpoint2.PrivateIPAddresses, - CustomNetworkInterfaceName: "test-if-name", - }, - existing: armnetwork.PrivateEndpoint{ - Name: ptr.To("test-private-endpoint2"), - Location: ptr.To("test-location"), - Properties: &armnetwork.PrivateEndpointProperties{ - Subnet: &armnetwork.Subnet{ - ID: ptr.To("test-subnet"), - }, - ApplicationSecurityGroups: []*armnetwork.ApplicationSecurityGroup{{ - ID: ptr.To("asg1"), - }}, - PrivateLinkServiceConnections: []*armnetwork.PrivateLinkServiceConnection{{ - Name: ptr.To(privateEndpoint1.PrivateLinkServiceConnections[0].Name), - Properties: &armnetwork.PrivateLinkServiceConnectionProperties{ - PrivateLinkServiceID: ptr.To(privateEndpoint1.PrivateLinkServiceConnections[0].PrivateLinkServiceID), - GroupIDs: []*string{ptr.To("aa"), ptr.To("bb")}, - RequestMessage: ptr.To(privateEndpoint1.PrivateLinkServiceConnections[0].RequestMessage), - }, - }}, - ManualPrivateLinkServiceConnections: []*armnetwork.PrivateLinkServiceConnection{}, - ProvisioningState: ptr.To(armnetwork.ProvisioningStateSucceeded), - IPConfigurations: []*armnetwork.PrivateEndpointIPConfiguration{ + name: "user updates ASO's private endpoint resource and capz should overwrite it", + spec: ptr.To(fakePrivateEndpoint), + existing: getASOPrivateEndpoint( + // add ExtendedLocation + func(endpoint *asonetworkv1.PrivateEndpoint) { + endpoint.Spec.ExtendedLocation = fakeExtendedLocation.DeepCopy() + }, + + // User also updates private IP addresses and location. + // This change should be overwritten by CAPZ. + func(endpoint *asonetworkv1.PrivateEndpoint) { + endpoint.Spec.IpConfigurations = []asonetworkv1.PrivateEndpointIPConfiguration{ { - Properties: &armnetwork.PrivateEndpointIPConfigurationProperties{ - PrivateIPAddress: ptr.To("10.0.0.1"), - }, + PrivateIPAddress: ptr.To("9.9.9.9"), }, - }, - CustomNetworkInterfaceName: ptr.To("test-if-name"), + } + endpoint.Spec.Location = ptr.To("new_location") }, - Tags: map[string]*string{"sigs.k8s.io_cluster-api-provider-azure_cluster_my-cluster": ptr.To("owned"), "Name": ptr.To("test-private-endpoint2")}, - }, - expect: func(g *WithT, result interface{}) { - g.Expect(result).To(BeAssignableToTypeOf(armnetwork.PrivateEndpoint{})) - g.Expect(result).To(Equal(armnetwork.PrivateEndpoint{ - Name: ptr.To("test-private-endpoint2"), - Location: ptr.To("test-location"), - Properties: &armnetwork.PrivateEndpointProperties{ - Subnet: &armnetwork.Subnet{ - ID: ptr.To("test-subnet"), - Properties: &armnetwork.SubnetPropertiesFormat{ - PrivateEndpointNetworkPolicies: ptr.To(armnetwork.VirtualNetworkPrivateEndpointNetworkPoliciesDisabled), - PrivateLinkServiceNetworkPolicies: ptr.To(armnetwork.VirtualNetworkPrivateLinkServiceNetworkPoliciesEnabled), - }, - }, - ApplicationSecurityGroups: []*armnetwork.ApplicationSecurityGroup{{ - ID: ptr.To("asg1"), - }}, - PrivateLinkServiceConnections: []*armnetwork.PrivateLinkServiceConnection{{ - Name: ptr.To(privateEndpoint1.PrivateLinkServiceConnections[0].Name), - Properties: &armnetwork.PrivateLinkServiceConnectionProperties{ - PrivateLinkServiceID: ptr.To(privateEndpoint1.PrivateLinkServiceConnections[0].PrivateLinkServiceID), - GroupIDs: []*string{ptr.To("aa"), ptr.To("bb")}, - RequestMessage: ptr.To(privateEndpoint1.PrivateLinkServiceConnections[0].RequestMessage), - }, - }}, - ManualPrivateLinkServiceConnections: []*armnetwork.PrivateLinkServiceConnection{}, - IPConfigurations: []*armnetwork.PrivateEndpointIPConfiguration{ - { - Properties: &armnetwork.PrivateEndpointIPConfigurationProperties{ - PrivateIPAddress: ptr.To("10.0.0.1"), - }, - }, - { - Properties: &armnetwork.PrivateEndpointIPConfigurationProperties{ - PrivateIPAddress: ptr.To("10.0.0.2"), - }, - }, - }, - CustomNetworkInterfaceName: ptr.To("test-if-name"), - }, - Tags: map[string]*string{"sigs.k8s.io_cluster-api-provider-azure_cluster_my-cluster": ptr.To("owned"), "Name": ptr.To("test-private-endpoint2")}, - })) - }, - }, - { - name: "PrivateEndpoint doesn't exist", - spec: &PrivateEndpointSpec{ - Name: privateEndpoint2.Name, - Location: privateEndpoint2.Location, - ResourceGroup: "test-group", - ClusterName: "my-cluster", - ApplicationSecurityGroups: privateEndpoint2.ApplicationSecurityGroups, - PrivateLinkServiceConnections: []PrivateLinkServiceConnection{{ - Name: privateEndpoint2.PrivateLinkServiceConnections[0].Name, - GroupIDs: privateEndpoint2.PrivateLinkServiceConnections[0].GroupIDs, - PrivateLinkServiceID: privateEndpoint2.PrivateLinkServiceConnections[0].PrivateLinkServiceID, - RequestMessage: privateEndpoint2.PrivateLinkServiceConnections[0].RequestMessage, - }}, - SubnetID: "test-subnet", - PrivateIPAddresses: privateEndpoint2.PrivateIPAddresses, - CustomNetworkInterfaceName: "test-if-name", - }, - existing: nil, - expect: func(g *WithT, result interface{}) { - g.Expect(result).To(BeAssignableToTypeOf(armnetwork.PrivateEndpoint{})) - g.Expect(result).To(Equal(armnetwork.PrivateEndpoint{ - Name: ptr.To("test-private-endpoint2"), - Location: ptr.To("test-location"), - Properties: &armnetwork.PrivateEndpointProperties{ - Subnet: &armnetwork.Subnet{ - ID: ptr.To("test-subnet"), - Properties: &armnetwork.SubnetPropertiesFormat{ - PrivateEndpointNetworkPolicies: ptr.To(armnetwork.VirtualNetworkPrivateEndpointNetworkPoliciesDisabled), - PrivateLinkServiceNetworkPolicies: ptr.To(armnetwork.VirtualNetworkPrivateLinkServiceNetworkPoliciesEnabled), - }, - }, - ApplicationSecurityGroups: []*armnetwork.ApplicationSecurityGroup{{ - ID: ptr.To("asg1"), - }}, - PrivateLinkServiceConnections: []*armnetwork.PrivateLinkServiceConnection{{ - Name: ptr.To(privateEndpoint1.PrivateLinkServiceConnections[0].Name), - Properties: &armnetwork.PrivateLinkServiceConnectionProperties{ - PrivateLinkServiceID: ptr.To(privateEndpoint1.PrivateLinkServiceConnections[0].PrivateLinkServiceID), - GroupIDs: []*string{ptr.To("aa"), ptr.To("bb")}, - RequestMessage: ptr.To(privateEndpoint1.PrivateLinkServiceConnections[0].RequestMessage), - }, - }}, - ManualPrivateLinkServiceConnections: []*armnetwork.PrivateLinkServiceConnection{}, - IPConfigurations: []*armnetwork.PrivateEndpointIPConfiguration{ - { - Properties: &armnetwork.PrivateEndpointIPConfigurationProperties{ - PrivateIPAddress: ptr.To("10.0.0.1"), - }, - }, - { - Properties: &armnetwork.PrivateEndpointIPConfigurationProperties{ - PrivateIPAddress: ptr.To("10.0.0.2"), - }, - }, - }, - CustomNetworkInterfaceName: ptr.To("test-if-name"), + // add Status + func(endpoint *asonetworkv1.PrivateEndpoint) { + endpoint.Status = fakeASOPrivateEndpointsStatus + }, + ), + expect: func(g *WithT, result asonetworkv1.PrivateEndpoint) { + g.Expect(result).NotTo(BeNil()) + resultantASOPrivateEndpoint := getASOPrivateEndpoint( + func(endpoint *asonetworkv1.PrivateEndpoint) { + endpoint.Spec.ExtendedLocation = fakeExtendedLocation.DeepCopy() }, - Tags: map[string]*string{"sigs.k8s.io_cluster-api-provider-azure_cluster_my-cluster": ptr.To("owned"), "Name": ptr.To("test-private-endpoint2")}, - })) + ) + + // user changes except ExtendedLocation should be overwritten by CAPZ. + g.Expect(result.ObjectMeta).To(Equal(resultantASOPrivateEndpoint.ObjectMeta)) + + // Extended location addition is accepted. + g.Expect(result.Spec).To(Equal(resultantASOPrivateEndpoint.Spec)) + + // Status should be carried over. + g.Expect(result.Status).To(Equal(fakeASOPrivateEndpointsStatus)) }, }, } @@ -363,7 +236,7 @@ func TestParameters(t *testing.T) { } else { g.Expect(err).NotTo(HaveOccurred()) } - tc.expect(g, result) + tc.expect(g, *result) }) } } diff --git a/config/aso/crds.yaml b/config/aso/crds.yaml index caa89e77c16..47283e7db07 100644 --- a/config/aso/crds.yaml +++ b/config/aso/crds.yaml @@ -20265,6 +20265,996 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: azureserviceoperator-system/azureserviceoperator-serving-cert + controller-gen.kubebuilder.io/version: v0.13.0 + labels: + app.kubernetes.io/name: azure-service-operator + app.kubernetes.io/version: v2.4.0 + name: privateendpoints.network.azure.com +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: azureserviceoperator-webhook-service + namespace: azureserviceoperator-system + path: /convert + port: 443 + conversionReviewVersions: + - v1 + group: network.azure.com + names: + kind: PrivateEndpoint + listKind: PrivateEndpointList + plural: privateendpoints + singular: privateendpoint + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].severity + name: Severity + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].reason + name: Reason + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].message + name: Message + type: string + name: v1api20220701 + schema: + openAPIV3Schema: + description: 'Generator information: - Generated from: /network/resource-manager/Microsoft.Network/stable/2022-07-01/privateEndpoint.json - ARM URI: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/privateEndpoints/{privateEndpointName}' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + applicationSecurityGroups: + description: 'ApplicationSecurityGroups: Application security groups in which the private endpoint IP configuration is included.' + items: + description: An application security group in a resource group. + properties: + reference: + description: 'Reference: Resource ID.' + properties: + armId: + description: ARMID is a string of the form /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}. The /resourcegroups/{resourceGroupName} bit is optional as some resources are scoped at the subscription level ARMID is mutually exclusive with Group, Kind, Namespace and Name. + pattern: (?i)(^(/subscriptions/([^/]+)(/resourcegroups/([^/]+))?)?/providers/([^/]+)/([^/]+/[^/]+)(/([^/]+/[^/]+))*$|^/subscriptions/([^/]+)(/resourcegroups/([^/]+))?$) + type: string + group: + description: Group is the Kubernetes group of the resource. + type: string + kind: + description: Kind is the Kubernetes kind of the resource. + type: string + name: + description: Name is the Kubernetes name of the resource. + type: string + type: object + type: object + type: array + azureName: + description: 'AzureName: The name of the resource in Azure. This is often the same as the name of the resource in Kubernetes but it doesn''t have to be.' + type: string + customNetworkInterfaceName: + description: 'CustomNetworkInterfaceName: The custom name of the network interface attached to the private endpoint.' + type: string + extendedLocation: + description: 'ExtendedLocation: The extended location of the load balancer.' + properties: + name: + description: 'Name: The name of the extended location.' + type: string + type: + description: 'Type: The type of the extended location.' + enum: + - EdgeZone + type: string + type: object + ipConfigurations: + description: 'IpConfigurations: A list of IP configurations of the private endpoint. This will be used to map to the First Party Service''s endpoints.' + items: + description: An IP Configuration of the private endpoint. + properties: + groupId: + description: 'GroupId: The ID of a group obtained from the remote resource that this private endpoint should connect to.' + type: string + memberName: + description: 'MemberName: The member name of a group obtained from the remote resource that this private endpoint should connect to.' + type: string + name: + description: 'Name: The name of the resource that is unique within a resource group.' + type: string + privateIPAddress: + description: 'PrivateIPAddress: A private ip address obtained from the private endpoint''s subnet.' + type: string + type: object + type: array + location: + description: 'Location: Resource location.' + type: string + manualPrivateLinkServiceConnections: + description: 'ManualPrivateLinkServiceConnections: A grouping of information about the connection to the remote resource. Used when the network admin does not have access to approve connections to the remote resource.' + items: + description: PrivateLinkServiceConnection resource. + properties: + groupIds: + description: 'GroupIds: The ID(s) of the group(s) obtained from the remote resource that this private endpoint should connect to.' + items: + type: string + type: array + name: + description: 'Name: The name of the resource that is unique within a resource group. This name can be used to access the resource.' + type: string + privateLinkServiceConnectionState: + description: 'PrivateLinkServiceConnectionState: A collection of read-only information about the state of the connection to the remote resource.' + properties: + actionsRequired: + description: 'ActionsRequired: A message indicating if changes on the service provider require any updates on the consumer.' + type: string + description: + description: 'Description: The reason for approval/rejection of the connection.' + type: string + status: + description: 'Status: Indicates whether the connection has been Approved/Rejected/Removed by the owner of the service.' + type: string + type: object + privateLinkServiceReference: + description: 'PrivateLinkServiceReference: The resource id of private link service.' + properties: + armId: + description: ARMID is a string of the form /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}. The /resourcegroups/{resourceGroupName} bit is optional as some resources are scoped at the subscription level ARMID is mutually exclusive with Group, Kind, Namespace and Name. + pattern: (?i)(^(/subscriptions/([^/]+)(/resourcegroups/([^/]+))?)?/providers/([^/]+)/([^/]+/[^/]+)(/([^/]+/[^/]+))*$|^/subscriptions/([^/]+)(/resourcegroups/([^/]+))?$) + type: string + group: + description: Group is the Kubernetes group of the resource. + type: string + kind: + description: Kind is the Kubernetes kind of the resource. + type: string + name: + description: Name is the Kubernetes name of the resource. + type: string + type: object + requestMessage: + description: 'RequestMessage: A message passed to the owner of the remote resource with this connection request. Restricted to 140 chars.' + type: string + type: object + type: array + owner: + description: 'Owner: The owner of the resource. The owner controls where the resource goes when it is deployed. The owner also controls the resources lifecycle. When the owner is deleted the resource will also be deleted. Owner is expected to be a reference to a resources.azure.com/ResourceGroup resource' + properties: + armId: + pattern: (?i)(^(/subscriptions/([^/]+)(/resourcegroups/([^/]+))?)?/providers/([^/]+)/([^/]+/[^/]+)(/([^/]+/[^/]+))*$|^/subscriptions/([^/]+)(/resourcegroups/([^/]+))?$) + type: string + name: + description: This is the name of the Kubernetes resource to reference. + type: string + type: object + privateLinkServiceConnections: + description: 'PrivateLinkServiceConnections: A grouping of information about the connection to the remote resource.' + items: + description: PrivateLinkServiceConnection resource. + properties: + groupIds: + description: 'GroupIds: The ID(s) of the group(s) obtained from the remote resource that this private endpoint should connect to.' + items: + type: string + type: array + name: + description: 'Name: The name of the resource that is unique within a resource group. This name can be used to access the resource.' + type: string + privateLinkServiceConnectionState: + description: 'PrivateLinkServiceConnectionState: A collection of read-only information about the state of the connection to the remote resource.' + properties: + actionsRequired: + description: 'ActionsRequired: A message indicating if changes on the service provider require any updates on the consumer.' + type: string + description: + description: 'Description: The reason for approval/rejection of the connection.' + type: string + status: + description: 'Status: Indicates whether the connection has been Approved/Rejected/Removed by the owner of the service.' + type: string + type: object + privateLinkServiceReference: + description: 'PrivateLinkServiceReference: The resource id of private link service.' + properties: + armId: + description: ARMID is a string of the form /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}. The /resourcegroups/{resourceGroupName} bit is optional as some resources are scoped at the subscription level ARMID is mutually exclusive with Group, Kind, Namespace and Name. + pattern: (?i)(^(/subscriptions/([^/]+)(/resourcegroups/([^/]+))?)?/providers/([^/]+)/([^/]+/[^/]+)(/([^/]+/[^/]+))*$|^/subscriptions/([^/]+)(/resourcegroups/([^/]+))?$) + type: string + group: + description: Group is the Kubernetes group of the resource. + type: string + kind: + description: Kind is the Kubernetes kind of the resource. + type: string + name: + description: Name is the Kubernetes name of the resource. + type: string + type: object + requestMessage: + description: 'RequestMessage: A message passed to the owner of the remote resource with this connection request. Restricted to 140 chars.' + type: string + type: object + type: array + subnet: + description: 'Subnet: The ID of the subnet from which the private IP will be allocated.' + properties: + reference: + description: 'Reference: Resource ID.' + properties: + armId: + description: ARMID is a string of the form /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}. The /resourcegroups/{resourceGroupName} bit is optional as some resources are scoped at the subscription level ARMID is mutually exclusive with Group, Kind, Namespace and Name. + pattern: (?i)(^(/subscriptions/([^/]+)(/resourcegroups/([^/]+))?)?/providers/([^/]+)/([^/]+/[^/]+)(/([^/]+/[^/]+))*$|^/subscriptions/([^/]+)(/resourcegroups/([^/]+))?$) + type: string + group: + description: Group is the Kubernetes group of the resource. + type: string + kind: + description: Kind is the Kubernetes kind of the resource. + type: string + name: + description: Name is the Kubernetes name of the resource. + type: string + type: object + type: object + tags: + additionalProperties: + type: string + description: 'Tags: Resource tags.' + type: object + required: + - owner + type: object + status: + description: Private endpoint resource. + properties: + applicationSecurityGroups: + description: 'ApplicationSecurityGroups: Application security groups in which the private endpoint IP configuration is included.' + items: + description: An application security group in a resource group. + properties: + id: + description: 'Id: Resource ID.' + type: string + type: object + type: array + conditions: + description: 'Conditions: The observed state of the resource' + items: + description: Condition defines an extension to status (an observation) of a resource + properties: + lastTransitionTime: + description: LastTransitionTime is the last time the condition transitioned from one status to another. + format: date-time + type: string + message: + description: Message is a human readable message indicating details about the transition. This field may be empty. + type: string + observedGeneration: + description: ObservedGeneration is the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. + format: int64 + type: integer + reason: + description: Reason for the condition's last transition. Reasons are upper CamelCase (PascalCase) with no spaces. A reason is always provided, this field will not be empty. + type: string + severity: + description: Severity with which to treat failures of this type of condition. For conditions which have positive polarity (Status == True is their normal/healthy state), this will be omitted when Status == True For conditions which have negative polarity (Status == False is their normal/healthy state), this will be omitted when Status == False. This is omitted in all cases when Status == Unknown + type: string + status: + description: Status of the condition, one of True, False, or Unknown. + type: string + type: + description: Type of condition. + type: string + required: + - lastTransitionTime + - reason + - status + - type + type: object + type: array + customDnsConfigs: + description: 'CustomDnsConfigs: An array of custom dns configurations.' + items: + description: Contains custom Dns resolution configuration from customer. + properties: + fqdn: + description: 'Fqdn: Fqdn that resolves to private endpoint ip address.' + type: string + ipAddresses: + description: 'IpAddresses: A list of private ip addresses of the private endpoint.' + items: + type: string + type: array + type: object + type: array + customNetworkInterfaceName: + description: 'CustomNetworkInterfaceName: The custom name of the network interface attached to the private endpoint.' + type: string + etag: + description: 'Etag: A unique read-only string that changes whenever the resource is updated.' + type: string + extendedLocation: + description: 'ExtendedLocation: The extended location of the load balancer.' + properties: + name: + description: 'Name: The name of the extended location.' + type: string + type: + description: 'Type: The type of the extended location.' + type: string + type: object + id: + description: 'Id: Resource ID.' + type: string + ipConfigurations: + description: 'IpConfigurations: A list of IP configurations of the private endpoint. This will be used to map to the First Party Service''s endpoints.' + items: + description: An IP Configuration of the private endpoint. + properties: + etag: + description: 'Etag: A unique read-only string that changes whenever the resource is updated.' + type: string + groupId: + description: 'GroupId: The ID of a group obtained from the remote resource that this private endpoint should connect to.' + type: string + memberName: + description: 'MemberName: The member name of a group obtained from the remote resource that this private endpoint should connect to.' + type: string + name: + description: 'Name: The name of the resource that is unique within a resource group.' + type: string + privateIPAddress: + description: 'PrivateIPAddress: A private ip address obtained from the private endpoint''s subnet.' + type: string + type: + description: 'Type: The resource type.' + type: string + type: object + type: array + location: + description: 'Location: Resource location.' + type: string + manualPrivateLinkServiceConnections: + description: 'ManualPrivateLinkServiceConnections: A grouping of information about the connection to the remote resource. Used when the network admin does not have access to approve connections to the remote resource.' + items: + description: PrivateLinkServiceConnection resource. + properties: + etag: + description: 'Etag: A unique read-only string that changes whenever the resource is updated.' + type: string + groupIds: + description: 'GroupIds: The ID(s) of the group(s) obtained from the remote resource that this private endpoint should connect to.' + items: + type: string + type: array + id: + description: 'Id: Resource ID.' + type: string + name: + description: 'Name: The name of the resource that is unique within a resource group. This name can be used to access the resource.' + type: string + privateLinkServiceConnectionState: + description: 'PrivateLinkServiceConnectionState: A collection of read-only information about the state of the connection to the remote resource.' + properties: + actionsRequired: + description: 'ActionsRequired: A message indicating if changes on the service provider require any updates on the consumer.' + type: string + description: + description: 'Description: The reason for approval/rejection of the connection.' + type: string + status: + description: 'Status: Indicates whether the connection has been Approved/Rejected/Removed by the owner of the service.' + type: string + type: object + privateLinkServiceId: + description: 'PrivateLinkServiceId: The resource id of private link service.' + type: string + provisioningState: + description: 'ProvisioningState: The provisioning state of the private link service connection resource.' + type: string + requestMessage: + description: 'RequestMessage: A message passed to the owner of the remote resource with this connection request. Restricted to 140 chars.' + type: string + type: + description: 'Type: The resource type.' + type: string + type: object + type: array + name: + description: 'Name: Resource name.' + type: string + networkInterfaces: + description: 'NetworkInterfaces: An array of references to the network interfaces created for this private endpoint.' + items: + description: A network interface in a resource group. + properties: + id: + description: 'Id: Resource ID.' + type: string + type: object + type: array + privateLinkServiceConnections: + description: 'PrivateLinkServiceConnections: A grouping of information about the connection to the remote resource.' + items: + description: PrivateLinkServiceConnection resource. + properties: + etag: + description: 'Etag: A unique read-only string that changes whenever the resource is updated.' + type: string + groupIds: + description: 'GroupIds: The ID(s) of the group(s) obtained from the remote resource that this private endpoint should connect to.' + items: + type: string + type: array + id: + description: 'Id: Resource ID.' + type: string + name: + description: 'Name: The name of the resource that is unique within a resource group. This name can be used to access the resource.' + type: string + privateLinkServiceConnectionState: + description: 'PrivateLinkServiceConnectionState: A collection of read-only information about the state of the connection to the remote resource.' + properties: + actionsRequired: + description: 'ActionsRequired: A message indicating if changes on the service provider require any updates on the consumer.' + type: string + description: + description: 'Description: The reason for approval/rejection of the connection.' + type: string + status: + description: 'Status: Indicates whether the connection has been Approved/Rejected/Removed by the owner of the service.' + type: string + type: object + privateLinkServiceId: + description: 'PrivateLinkServiceId: The resource id of private link service.' + type: string + provisioningState: + description: 'ProvisioningState: The provisioning state of the private link service connection resource.' + type: string + requestMessage: + description: 'RequestMessage: A message passed to the owner of the remote resource with this connection request. Restricted to 140 chars.' + type: string + type: + description: 'Type: The resource type.' + type: string + type: object + type: array + provisioningState: + description: 'ProvisioningState: The provisioning state of the private endpoint resource.' + type: string + subnet: + description: 'Subnet: The ID of the subnet from which the private IP will be allocated.' + properties: + id: + description: 'Id: Resource ID.' + type: string + type: object + tags: + additionalProperties: + type: string + description: 'Tags: Resource tags.' + type: object + type: + description: 'Type: Resource type.' + type: string + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].severity + name: Severity + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].reason + name: Reason + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].message + name: Message + type: string + name: v1api20220701storage + schema: + openAPIV3Schema: + description: 'Storage version of v1api20220701.PrivateEndpoint Generator information: - Generated from: /network/resource-manager/Microsoft.Network/stable/2022-07-01/privateEndpoint.json - ARM URI: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/privateEndpoints/{privateEndpointName}' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Storage version of v1api20220701.PrivateEndpoint_Spec + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + applicationSecurityGroups: + items: + description: Storage version of v1api20220701.ApplicationSecurityGroupSpec_PrivateEndpoint_SubResourceEmbedded An application security group in a resource group. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + reference: + description: 'Reference: Resource ID.' + properties: + armId: + description: ARMID is a string of the form /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}. The /resourcegroups/{resourceGroupName} bit is optional as some resources are scoped at the subscription level ARMID is mutually exclusive with Group, Kind, Namespace and Name. + pattern: (?i)(^(/subscriptions/([^/]+)(/resourcegroups/([^/]+))?)?/providers/([^/]+)/([^/]+/[^/]+)(/([^/]+/[^/]+))*$|^/subscriptions/([^/]+)(/resourcegroups/([^/]+))?$) + type: string + group: + description: Group is the Kubernetes group of the resource. + type: string + kind: + description: Kind is the Kubernetes kind of the resource. + type: string + name: + description: Name is the Kubernetes name of the resource. + type: string + type: object + type: object + type: array + azureName: + description: 'AzureName: The name of the resource in Azure. This is often the same as the name of the resource in Kubernetes but it doesn''t have to be.' + type: string + customNetworkInterfaceName: + type: string + extendedLocation: + description: Storage version of v1api20220701.ExtendedLocation ExtendedLocation complex type. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + name: + type: string + type: + type: string + type: object + ipConfigurations: + items: + description: Storage version of v1api20220701.PrivateEndpointIPConfiguration An IP Configuration of the private endpoint. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + groupId: + type: string + memberName: + type: string + name: + type: string + privateIPAddress: + type: string + type: object + type: array + location: + type: string + manualPrivateLinkServiceConnections: + items: + description: Storage version of v1api20220701.PrivateLinkServiceConnection PrivateLinkServiceConnection resource. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + groupIds: + items: + type: string + type: array + name: + type: string + privateLinkServiceConnectionState: + description: Storage version of v1api20220701.PrivateLinkServiceConnectionState A collection of information about the state of the connection between service consumer and provider. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + actionsRequired: + type: string + description: + type: string + status: + type: string + type: object + privateLinkServiceReference: + description: 'PrivateLinkServiceReference: The resource id of private link service.' + properties: + armId: + description: ARMID is a string of the form /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}. The /resourcegroups/{resourceGroupName} bit is optional as some resources are scoped at the subscription level ARMID is mutually exclusive with Group, Kind, Namespace and Name. + pattern: (?i)(^(/subscriptions/([^/]+)(/resourcegroups/([^/]+))?)?/providers/([^/]+)/([^/]+/[^/]+)(/([^/]+/[^/]+))*$|^/subscriptions/([^/]+)(/resourcegroups/([^/]+))?$) + type: string + group: + description: Group is the Kubernetes group of the resource. + type: string + kind: + description: Kind is the Kubernetes kind of the resource. + type: string + name: + description: Name is the Kubernetes name of the resource. + type: string + type: object + requestMessage: + type: string + type: object + type: array + originalVersion: + type: string + owner: + description: 'Owner: The owner of the resource. The owner controls where the resource goes when it is deployed. The owner also controls the resources lifecycle. When the owner is deleted the resource will also be deleted. Owner is expected to be a reference to a resources.azure.com/ResourceGroup resource' + properties: + armId: + pattern: (?i)(^(/subscriptions/([^/]+)(/resourcegroups/([^/]+))?)?/providers/([^/]+)/([^/]+/[^/]+)(/([^/]+/[^/]+))*$|^/subscriptions/([^/]+)(/resourcegroups/([^/]+))?$) + type: string + name: + description: This is the name of the Kubernetes resource to reference. + type: string + type: object + privateLinkServiceConnections: + items: + description: Storage version of v1api20220701.PrivateLinkServiceConnection PrivateLinkServiceConnection resource. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + groupIds: + items: + type: string + type: array + name: + type: string + privateLinkServiceConnectionState: + description: Storage version of v1api20220701.PrivateLinkServiceConnectionState A collection of information about the state of the connection between service consumer and provider. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + actionsRequired: + type: string + description: + type: string + status: + type: string + type: object + privateLinkServiceReference: + description: 'PrivateLinkServiceReference: The resource id of private link service.' + properties: + armId: + description: ARMID is a string of the form /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}. The /resourcegroups/{resourceGroupName} bit is optional as some resources are scoped at the subscription level ARMID is mutually exclusive with Group, Kind, Namespace and Name. + pattern: (?i)(^(/subscriptions/([^/]+)(/resourcegroups/([^/]+))?)?/providers/([^/]+)/([^/]+/[^/]+)(/([^/]+/[^/]+))*$|^/subscriptions/([^/]+)(/resourcegroups/([^/]+))?$) + type: string + group: + description: Group is the Kubernetes group of the resource. + type: string + kind: + description: Kind is the Kubernetes kind of the resource. + type: string + name: + description: Name is the Kubernetes name of the resource. + type: string + type: object + requestMessage: + type: string + type: object + type: array + subnet: + description: Storage version of v1api20220701.Subnet_PrivateEndpoint_SubResourceEmbedded Subnet in a virtual network resource. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + reference: + description: 'Reference: Resource ID.' + properties: + armId: + description: ARMID is a string of the form /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}. The /resourcegroups/{resourceGroupName} bit is optional as some resources are scoped at the subscription level ARMID is mutually exclusive with Group, Kind, Namespace and Name. + pattern: (?i)(^(/subscriptions/([^/]+)(/resourcegroups/([^/]+))?)?/providers/([^/]+)/([^/]+/[^/]+)(/([^/]+/[^/]+))*$|^/subscriptions/([^/]+)(/resourcegroups/([^/]+))?$) + type: string + group: + description: Group is the Kubernetes group of the resource. + type: string + kind: + description: Kind is the Kubernetes kind of the resource. + type: string + name: + description: Name is the Kubernetes name of the resource. + type: string + type: object + type: object + tags: + additionalProperties: + type: string + type: object + required: + - owner + type: object + status: + description: Storage version of v1api20220701.PrivateEndpoint_STATUS_PrivateEndpoint_SubResourceEmbedded Private endpoint resource. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + applicationSecurityGroups: + items: + description: Storage version of v1api20220701.ApplicationSecurityGroup_STATUS_PrivateEndpoint_SubResourceEmbedded An application security group in a resource group. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + id: + type: string + type: object + type: array + conditions: + items: + description: Condition defines an extension to status (an observation) of a resource + properties: + lastTransitionTime: + description: LastTransitionTime is the last time the condition transitioned from one status to another. + format: date-time + type: string + message: + description: Message is a human readable message indicating details about the transition. This field may be empty. + type: string + observedGeneration: + description: ObservedGeneration is the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. + format: int64 + type: integer + reason: + description: Reason for the condition's last transition. Reasons are upper CamelCase (PascalCase) with no spaces. A reason is always provided, this field will not be empty. + type: string + severity: + description: Severity with which to treat failures of this type of condition. For conditions which have positive polarity (Status == True is their normal/healthy state), this will be omitted when Status == True For conditions which have negative polarity (Status == False is their normal/healthy state), this will be omitted when Status == False. This is omitted in all cases when Status == Unknown + type: string + status: + description: Status of the condition, one of True, False, or Unknown. + type: string + type: + description: Type of condition. + type: string + required: + - lastTransitionTime + - reason + - status + - type + type: object + type: array + customDnsConfigs: + items: + description: Storage version of v1api20220701.CustomDnsConfigPropertiesFormat_STATUS Contains custom Dns resolution configuration from customer. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + fqdn: + type: string + ipAddresses: + items: + type: string + type: array + type: object + type: array + customNetworkInterfaceName: + type: string + etag: + type: string + extendedLocation: + description: Storage version of v1api20220701.ExtendedLocation_STATUS ExtendedLocation complex type. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + name: + type: string + type: + type: string + type: object + id: + type: string + ipConfigurations: + items: + description: Storage version of v1api20220701.PrivateEndpointIPConfiguration_STATUS An IP Configuration of the private endpoint. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + etag: + type: string + groupId: + type: string + memberName: + type: string + name: + type: string + privateIPAddress: + type: string + type: + type: string + type: object + type: array + location: + type: string + manualPrivateLinkServiceConnections: + items: + description: Storage version of v1api20220701.PrivateLinkServiceConnection_STATUS PrivateLinkServiceConnection resource. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + etag: + type: string + groupIds: + items: + type: string + type: array + id: + type: string + name: + type: string + privateLinkServiceConnectionState: + description: Storage version of v1api20220701.PrivateLinkServiceConnectionState_STATUS A collection of information about the state of the connection between service consumer and provider. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + actionsRequired: + type: string + description: + type: string + status: + type: string + type: object + privateLinkServiceId: + type: string + provisioningState: + type: string + requestMessage: + type: string + type: + type: string + type: object + type: array + name: + type: string + networkInterfaces: + items: + description: Storage version of v1api20220701.NetworkInterface_STATUS_PrivateEndpoint_SubResourceEmbedded A network interface in a resource group. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + id: + type: string + type: object + type: array + privateLinkServiceConnections: + items: + description: Storage version of v1api20220701.PrivateLinkServiceConnection_STATUS PrivateLinkServiceConnection resource. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + etag: + type: string + groupIds: + items: + type: string + type: array + id: + type: string + name: + type: string + privateLinkServiceConnectionState: + description: Storage version of v1api20220701.PrivateLinkServiceConnectionState_STATUS A collection of information about the state of the connection between service consumer and provider. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + actionsRequired: + type: string + description: + type: string + status: + type: string + type: object + privateLinkServiceId: + type: string + provisioningState: + type: string + requestMessage: + type: string + type: + type: string + type: object + type: array + provisioningState: + type: string + subnet: + description: Storage version of v1api20220701.Subnet_STATUS_PrivateEndpoint_SubResourceEmbedded Subnet in a virtual network resource. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + id: + type: string + type: object + tags: + additionalProperties: + type: string + type: object + type: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: azureserviceoperator-system/azureserviceoperator-serving-cert diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 71052af80cc..df0a3d2f3b8 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -307,6 +307,7 @@ rules: resources: - bastionhosts - natgateways + - privateendpoints verbs: - create - delete @@ -320,6 +321,27 @@ rules: resources: - bastionhosts/status - natgateways/status + - privateendpoints/status + verbs: + - get + - list + - watch +- apiGroups: + - network.azure.com + resources: + - privateendpoints + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - network.azure.com + resources: + - privateendpoints/status verbs: - get - list diff --git a/controllers/azurecluster_controller.go b/controllers/azurecluster_controller.go index ebbe6b8abea..d1c4cb63509 100644 --- a/controllers/azurecluster_controller.go +++ b/controllers/azurecluster_controller.go @@ -113,8 +113,8 @@ func (acr *AzureClusterReconciler) SetupWithManager(ctx context.Context, mgr ctr // +kubebuilder:rbac:groups="",resources=namespaces,verbs=list; // +kubebuilder:rbac:groups=resources.azure.com,resources=resourcegroups,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=resources.azure.com,resources=resourcegroups/status,verbs=get;list;watch -// +kubebuilder:rbac:groups=network.azure.com,resources=natgateways;bastionhosts,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=network.azure.com,resources=natgateways/status;bastionhosts/status,verbs=get;list;watch +// +kubebuilder:rbac:groups=network.azure.com,resources=natgateways;bastionhosts;privateendpoints,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=network.azure.com,resources=natgateways/status;bastionhosts/status;privateendpoints/status,verbs=get;list;watch // Reconcile idempotently gets, creates, and updates a cluster. func (acr *AzureClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) { diff --git a/controllers/azurecluster_reconciler.go b/controllers/azurecluster_reconciler.go index 1ce2d9c233e..9fe78d2658d 100644 --- a/controllers/azurecluster_reconciler.go +++ b/controllers/azurecluster_reconciler.go @@ -65,10 +65,6 @@ func newAzureClusterService(scope *scope.ClusterScope) (*azureClusterService, er if err != nil { return nil, err } - privateEndpointsSvc, err := privateendpoints.New(scope) - if err != nil { - return nil, err - } publicIPsSvc, err := publicips.New(scope) if err != nil { return nil, err @@ -106,8 +102,8 @@ func newAzureClusterService(scope *scope.ClusterScope) (*azureClusterService, er vnetPeeringsSvc, loadbalancersSvc, privateDNSSvc, + privateendpoints.New(scope), bastionhosts.New(scope), - privateEndpointsSvc, }, skuCache: skuCache, } diff --git a/controllers/azuremanagedcontrolplane_controller.go b/controllers/azuremanagedcontrolplane_controller.go index f0e7453628a..bfbbb2f15bd 100644 --- a/controllers/azuremanagedcontrolplane_controller.go +++ b/controllers/azuremanagedcontrolplane_controller.go @@ -117,6 +117,8 @@ func (amcpr *AzureManagedControlPlaneReconciler) SetupWithManager(ctx context.Co // +kubebuilder:rbac:groups=resources.azure.com,resources=resourcegroups/status,verbs=get;list;watch // +kubebuilder:rbac:groups=containerservice.azure.com,resources=managedclusters,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=containerservice.azure.com,resources=managedclusters/status,verbs=get;list;watch +// +kubebuilder:rbac:groups=network.azure.com,resources=privateendpoints,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=network.azure.com,resources=privateendpoints/status,verbs=get;list;watch // Reconcile idempotently gets, creates, and updates a managed control plane. func (amcpr *AzureManagedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) { diff --git a/controllers/azuremanagedcontrolplane_reconciler.go b/controllers/azuremanagedcontrolplane_reconciler.go index 9294474f918..09a170f1c1f 100644 --- a/controllers/azuremanagedcontrolplane_reconciler.go +++ b/controllers/azuremanagedcontrolplane_reconciler.go @@ -45,10 +45,6 @@ type azureManagedControlPlaneService struct { // newAzureManagedControlPlaneReconciler populates all the services based on input scope. func newAzureManagedControlPlaneReconciler(scope *scope.ManagedControlPlaneScope) (*azureManagedControlPlaneService, error) { - privateEndpointsSvc, err := privateendpoints.New(scope) - if err != nil { - return nil, err - } resourceHealthSvc, err := resourcehealth.New(scope) if err != nil { return nil, err @@ -69,7 +65,7 @@ func newAzureManagedControlPlaneReconciler(scope *scope.ManagedControlPlaneScope virtualNetworksSvc, subnetsSvc, managedclusters.New(scope), - privateEndpointsSvc, + privateendpoints.New(scope), resourceHealthSvc, }, }, nil