diff --git a/controllers/azurecluster_controller_test.go b/controllers/azurecluster_controller_test.go index 57df3d6869b..070f90bfc93 100644 --- a/controllers/azurecluster_controller_test.go +++ b/controllers/azurecluster_controller_test.go @@ -19,20 +19,43 @@ package controllers import ( "context" "testing" + "time" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" asoresourcesv1 "github.com/Azure/azure-service-operator/v2/api/resources/v1api20200601" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-azure/azure" + "sigs.k8s.io/cluster-api-provider-azure/azure/scope" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/resourceskus" "sigs.k8s.io/cluster-api-provider-azure/internal/test" "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + capierrors "sigs.k8s.io/cluster-api/errors" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +type TestClusterReconcileInput struct { + createAzureClusterService func(*scope.ClusterScope) (*azureClusterService, error) + azureClusterOptions func(ac *infrav1.AzureCluster) + clusterScopeFailureReason capierrors.ClusterStatusError + cache *scope.ClusterCache + expectedResult reconcile.Result + expectedErr string + ready bool +} + +const ( + location = "westus2" ) var _ = Describe("AzureClusterReconciler", func() { @@ -58,6 +81,162 @@ var _ = Describe("AzureClusterReconciler", func() { }) }) +func TestAzureClusterReconcile(t *testing.T) { + g := NewWithT(t) + scheme, err := newScheme() + g.Expect(err).NotTo(HaveOccurred()) + + defaultCluster := getFakeCluster() + defaultAzureCluster := getFakeAzureCluster() + + cases := map[string]struct { + objects []runtime.Object + fail bool + err string + event string + }{ + "should reconcile normally": { + objects: []runtime.Object{ + defaultCluster, + defaultAzureCluster, + }, + }, + "should raise event if the azure cluster is not found": { + objects: []runtime.Object{ + defaultCluster, + }, + event: "AzureClusterObjectNotFound", + }, + "should raise event if cluster is not found": { + objects: []runtime.Object{ + getFakeAzureCluster(func(ac *infrav1.AzureCluster) { + ac.OwnerReferences = nil + }), + defaultCluster, + }, + event: "OwnerRefNotSet", + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(tc.objects...). + WithStatusSubresource( + &infrav1.AzureCluster{}, + ). + Build() + + reconciler := &AzureClusterReconciler{ + Client: client, + Recorder: record.NewFakeRecorder(128), + } + + _, err := reconciler.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{ + Namespace: "default", + Name: "my-azure-cluster", + }, + }) + if tc.event != "" { + g.Expect(reconciler.Recorder.(*record.FakeRecorder).Events).To(Receive(ContainSubstring(tc.event))) + } + if tc.fail { + g.Expect(err).To(MatchError(tc.err)) + } else { + g.Expect(err).NotTo(HaveOccurred()) + } + }) + } +} + +func TestAzureClusterReconcileNormal(t *testing.T) { + cases := map[string]TestClusterReconcileInput{ + "should reconcile normally": { + createAzureClusterService: func(cs *scope.ClusterScope) (*azureClusterService, error) { + return getDefaultAzureClusterService(func(acs *azureClusterService) { + acs.skuCache = resourceskus.NewStaticCache([]armcompute.ResourceSKU{}, cs.Location()) + acs.scope = cs + }), nil + }, + cache: &scope.ClusterCache{}, + ready: true, + }, + "should fail if azure cluster service creator fails": { + createAzureClusterService: func(*scope.ClusterScope) (*azureClusterService, error) { + return nil, errors.New("failed to create azure cluster service") + }, + cache: &scope.ClusterCache{}, + expectedErr: "failed to create azure cluster service", + }, + "should reconcile if terminal error is received": { + createAzureClusterService: func(cs *scope.ClusterScope) (*azureClusterService, error) { + return getDefaultAzureClusterService(func(acs *azureClusterService) { + acs.skuCache = resourceskus.NewStaticCache([]armcompute.ResourceSKU{}, cs.Location()) + acs.scope = cs + }), nil + }, + clusterScopeFailureReason: capierrors.CreateClusterError, + cache: &scope.ClusterCache{}, + }, + "should requeue if transient error is received": { + createAzureClusterService: func(cs *scope.ClusterScope) (*azureClusterService, error) { + return getDefaultAzureClusterService(func(acs *azureClusterService) { + acs.skuCache = resourceskus.NewStaticCache([]armcompute.ResourceSKU{}, cs.Location()) + acs.scope = cs + acs.Reconcile = func(ctx context.Context) error { + return azure.WithTransientError(errors.New("failed to reconcile AzureCluster"), 10*time.Second) + } + }), nil + }, + cache: &scope.ClusterCache{}, + expectedResult: reconcile.Result{RequeueAfter: 10 * time.Second}, + }, + "should return error for general failures": { + createAzureClusterService: func(cs *scope.ClusterScope) (*azureClusterService, error) { + return getDefaultAzureClusterService(func(acs *azureClusterService) { + acs.skuCache = resourceskus.NewStaticCache([]armcompute.ResourceSKU{}, cs.Location()) + acs.scope = cs + acs.Reconcile = func(context.Context) error { + return errors.New("foo error") + } + acs.Pause = func(context.Context) error { + return errors.New("foo error") + } + acs.Delete = func(context.Context) error { + return errors.New("foo error") + } + }), nil + }, + cache: &scope.ClusterCache{}, + expectedErr: "failed to reconcile cluster services", + }, + } + + for name, c := range cases { + tc := c + t.Run(name, func(t *testing.T) { + g := NewWithT(t) + reconciler, clusterScope, err := getClusterReconcileInputs(tc) + g.Expect(err).NotTo(HaveOccurred()) + + result, err := reconciler.reconcileNormal(context.Background(), clusterScope) + g.Expect(result).To(Equal(tc.expectedResult)) + + if tc.ready { + g.Expect(clusterScope.AzureCluster.Status.Ready).To(BeTrue()) + } + if tc.expectedErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tc.expectedErr)) + } else { + g.Expect(err).NotTo(HaveOccurred()) + } + }) + } +} + func TestAzureClusterReconcilePaused(t *testing.T) { g := NewWithT(t) @@ -133,3 +312,148 @@ func TestAzureClusterReconcilePaused(t *testing.T) { g.Eventually(recorder.Events).Should(Receive(Equal("Normal ClusterPaused AzureCluster or linked Cluster is marked as paused. Won't reconcile normally"))) } + +func TestAzureClusterReconcileDelete(t *testing.T) { + cases := map[string]TestClusterReconcileInput{ + "should delete successfully": { + createAzureClusterService: func(cs *scope.ClusterScope) (*azureClusterService, error) { + return getDefaultAzureClusterService(func(acs *azureClusterService) { + acs.skuCache = resourceskus.NewStaticCache([]armcompute.ResourceSKU{}, cs.Location()) + acs.scope = cs + }), nil + }, + cache: &scope.ClusterCache{}, + }, + "should fail if failed to create azure cluster service": { + createAzureClusterService: func(cs *scope.ClusterScope) (*azureClusterService, error) { + return nil, errors.New("failed to create AzureClusterService") + }, + cache: &scope.ClusterCache{}, + expectedErr: "failed to create AzureClusterService", + }, + "should requeue if transient error is received": { + createAzureClusterService: func(cs *scope.ClusterScope) (*azureClusterService, error) { + return getDefaultAzureClusterService(func(acs *azureClusterService) { + acs.skuCache = resourceskus.NewStaticCache([]armcompute.ResourceSKU{}, cs.Location()) + acs.scope = cs + acs.Reconcile = func(ctx context.Context) error { + return azure.WithTransientError(errors.New("failed to reconcile AzureCluster"), 10*time.Second) + } + }), nil + }, + cache: &scope.ClusterCache{}, + expectedResult: reconcile.Result{}, + }, + "should fail to delete for non-transient errors": { + createAzureClusterService: func(cs *scope.ClusterScope) (*azureClusterService, error) { + return getDefaultAzureClusterService(func(acs *azureClusterService) { + acs.skuCache = resourceskus.NewStaticCache([]armcompute.ResourceSKU{}, cs.Location()) + acs.scope = cs + acs.Reconcile = func(context.Context) error { + return errors.New("foo error") + } + acs.Pause = func(context.Context) error { + return errors.New("foo error") + } + acs.Delete = func(context.Context) error { + return errors.New("foo error") + } + }), nil + }, + cache: &scope.ClusterCache{}, + expectedErr: "error deleting AzureCluster", + }, + } + + for name, c := range cases { + tc := c + t.Run(name, func(t *testing.T) { + g := NewWithT(t) + + reconciler, clusterScope, err := getClusterReconcileInputs(tc) + g.Expect(err).NotTo(HaveOccurred()) + + result, err := reconciler.reconcileDelete(context.Background(), clusterScope) + g.Expect(result).To(Equal(tc.expectedResult)) + + if tc.expectedErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tc.expectedErr)) + } else { + g.Expect(err).NotTo(HaveOccurred()) + } + }) + } +} + +func getDefaultAzureClusterService(changes ...func(*azureClusterService)) *azureClusterService { + input := &azureClusterService{ + services: []azure.ServiceReconciler{}, + Reconcile: func(ctx context.Context) error { + return nil + }, + Delete: func(ctx context.Context) error { + return nil + }, + Pause: func(ctx context.Context) error { + return nil + }, + } + + for _, change := range changes { + change(input) + } + + return input +} + +func getClusterReconcileInputs(tc TestClusterReconcileInput) (*AzureClusterReconciler, *scope.ClusterScope, error) { + scheme, err := newScheme() + if err != nil { + return nil, nil, err + } + + cluster := getFakeCluster() + + var azureCluster *infrav1.AzureCluster + if tc.azureClusterOptions != nil { + azureCluster = getFakeAzureCluster(tc.azureClusterOptions, func(ac *infrav1.AzureCluster) { + ac.Spec.Location = location + }) + } else { + azureCluster = getFakeAzureCluster(func(ac *infrav1.AzureCluster) { + ac.Spec.Location = location + }) + } + + objects := []runtime.Object{ + cluster, + azureCluster, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(objects...). + WithStatusSubresource( + &infrav1.AzureCluster{}, + ). + Build() + + reconciler := &AzureClusterReconciler{ + Client: client, + Recorder: record.NewFakeRecorder(128), + createAzureClusterService: tc.createAzureClusterService, + } + + clusterScope, err := scope.NewClusterScope(context.Background(), scope.ClusterScopeParams{ + Client: client, + Cluster: cluster, + AzureCluster: azureCluster, + Cache: tc.cache, + }) + if err != nil { + return nil, nil, err + } + + return reconciler, clusterScope, nil +} diff --git a/controllers/azurecluster_reconciler.go b/controllers/azurecluster_reconciler.go index d18fc5f3cca..28783f18c8f 100644 --- a/controllers/azurecluster_reconciler.go +++ b/controllers/azurecluster_reconciler.go @@ -44,8 +44,11 @@ type azureClusterService struct { scope *scope.ClusterScope // services is the list of services that are reconciled by this controller. // The order of the services is important as it determines the order in which the services are reconciled. - services []azure.ServiceReconciler - skuCache *resourceskus.Cache + services []azure.ServiceReconciler + skuCache *resourceskus.Cache + Reconcile func(context.Context) error + Pause func(context.Context) error + Delete func(context.Context) error } // newAzureClusterService populates all the services based on input scope. @@ -94,7 +97,7 @@ func newAzureClusterService(scope *scope.ClusterScope) (*azureClusterService, er if err != nil { return nil, err } - return &azureClusterService{ + acs := &azureClusterService{ scope: scope, services: []azure.ServiceReconciler{ groups.New(scope), @@ -111,11 +114,16 @@ func newAzureClusterService(scope *scope.ClusterScope) (*azureClusterService, er privateEndpointsSvc, }, skuCache: skuCache, - }, nil + } + acs.Reconcile = acs.reconcile + acs.Pause = acs.pause + acs.Delete = acs.delete + + return acs, nil } // Reconcile reconciles all the services in a predetermined order. -func (s *azureClusterService) Reconcile(ctx context.Context) error { +func (s *azureClusterService) reconcile(ctx context.Context) error { ctx, _, done := tele.StartSpanWithLogger(ctx, "controllers.azureClusterService.Reconcile") defer done() @@ -137,7 +145,7 @@ func (s *azureClusterService) Reconcile(ctx context.Context) error { } // Pause pauses all components making up the cluster. -func (s *azureClusterService) Pause(ctx context.Context) error { +func (s *azureClusterService) pause(ctx context.Context) error { ctx, _, done := tele.StartSpanWithLogger(ctx, "controllers.azureClusterService.Pause") defer done() @@ -155,7 +163,7 @@ func (s *azureClusterService) Pause(ctx context.Context) error { } // Delete reconciles all the services in a predetermined order. -func (s *azureClusterService) Delete(ctx context.Context) error { +func (s *azureClusterService) delete(ctx context.Context) error { ctx, _, done := tele.StartSpanWithLogger(ctx, "controllers.azureClusterService.Delete") defer done() diff --git a/controllers/azurecluster_reconciler_test.go b/controllers/azurecluster_reconciler_test.go index c0ee7709baf..0b37db367f8 100644 --- a/controllers/azurecluster_reconciler_test.go +++ b/controllers/azurecluster_reconciler_test.go @@ -94,7 +94,7 @@ func TestAzureClusterServiceReconcile(t *testing.T) { skuCache: resourceskus.NewStaticCache([]armcompute.ResourceSKU{}, ""), } - err := s.Reconcile(context.TODO()) + err := s.reconcile(context.TODO()) if tc.expectedError != "" { g.Expect(err).To(HaveOccurred()) g.Expect(err).To(MatchError(tc.expectedError)) @@ -164,7 +164,7 @@ func TestAzureClusterServicePause(t *testing.T) { }, } - err := s.Pause(context.TODO()) + err := s.pause(context.TODO()) if tc.expectedError != "" { g.Expect(err).To(HaveOccurred()) g.Expect(err).To(MatchError(tc.expectedError)) @@ -362,7 +362,7 @@ func TestAzureClusterServiceDelete(t *testing.T) { skuCache: resourceskus.NewStaticCache([]armcompute.ResourceSKU{}, ""), } - err := s.Delete(context.TODO()) + err := s.delete(context.TODO()) if tc.expectedError != "" { g.Expect(err).To(HaveOccurred()) g.Expect(err).To(MatchError(tc.expectedError)) diff --git a/controllers/azuremachine_controller_test.go b/controllers/azuremachine_controller_test.go index f5518adfabf..4d9c3199b49 100644 --- a/controllers/azuremachine_controller_test.go +++ b/controllers/azuremachine_controller_test.go @@ -41,7 +41,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -type TestReconcileInput struct { +type TestMachineReconcileInput struct { createAzureMachineService func(*scope.MachineScope) (*azureMachineService, error) azureMachineOptions func(am *infrav1.AzureMachine) expectedErr string @@ -161,7 +161,7 @@ func (f fakeSKUCacher) Get(context.Context, string, resourceskus.ResourceType) ( } func TestAzureMachineReconcileNormal(t *testing.T) { - cases := map[string]TestReconcileInput{ + cases := map[string]TestMachineReconcileInput{ "should reconcile normally": { createAzureMachineService: getFakeAzureMachineService, cache: &scope.MachineCache{}, @@ -229,7 +229,7 @@ func TestAzureMachineReconcileNormal(t *testing.T) { t.Run(name, func(t *testing.T) { g := NewWithT(t) - reconciler, machineScope, clusterScope, err := getReconcileInputs(tc) + reconciler, machineScope, clusterScope, err := getMachineReconcileInputs(tc) g.Expect(err).NotTo(HaveOccurred()) result, err := reconciler.reconcileNormal(context.Background(), machineScope, clusterScope) @@ -253,7 +253,7 @@ func TestAzureMachineReconcileNormal(t *testing.T) { } func TestAzureMachineReconcilePause(t *testing.T) { - cases := map[string]TestReconcileInput{ + cases := map[string]TestMachineReconcileInput{ "should pause successfully": { createAzureMachineService: getFakeAzureMachineService, cache: &scope.MachineCache{}, @@ -275,7 +275,7 @@ func TestAzureMachineReconcilePause(t *testing.T) { t.Run(name, func(t *testing.T) { g := NewWithT(t) - reconciler, machineScope, _, err := getReconcileInputs(tc) + reconciler, machineScope, _, err := getMachineReconcileInputs(tc) g.Expect(err).NotTo(HaveOccurred()) result, err := reconciler.reconcilePause(context.Background(), machineScope) @@ -292,7 +292,7 @@ func TestAzureMachineReconcilePause(t *testing.T) { } func TestAzureMachineReconcileDelete(t *testing.T) { - cases := map[string]TestReconcileInput{ + cases := map[string]TestMachineReconcileInput{ "should delete successfully": { createAzureMachineService: getFakeAzureMachineService, cache: &scope.MachineCache{}, @@ -319,7 +319,7 @@ func TestAzureMachineReconcileDelete(t *testing.T) { t.Run(name, func(t *testing.T) { g := NewWithT(t) - reconciler, machineScope, clusterScope, err := getReconcileInputs(tc) + reconciler, machineScope, clusterScope, err := getMachineReconcileInputs(tc) g.Expect(err).NotTo(HaveOccurred()) result, err := reconciler.reconcileDelete(context.Background(), machineScope, clusterScope) @@ -335,7 +335,7 @@ func TestAzureMachineReconcileDelete(t *testing.T) { } } -func getReconcileInputs(tc TestReconcileInput) (*AzureMachineReconciler, *scope.MachineScope, *scope.ClusterScope, error) { +func getMachineReconcileInputs(tc TestMachineReconcileInput) (*AzureMachineReconciler, *scope.MachineScope, *scope.ClusterScope, error) { scheme, err := newScheme() if err != nil { return nil, nil, nil, err @@ -507,8 +507,8 @@ func getDefaultAzureMachineService(machineScope *scope.MachineScope, cache *reso } } -func getFakeCluster(changes ...func(*clusterv1.Cluster)) *clusterv1.Cluster { - input := &clusterv1.Cluster{ +func getFakeCluster() *clusterv1.Cluster { + return &clusterv1.Cluster{ ObjectMeta: metav1.ObjectMeta{ Name: "my-cluster", Namespace: "default", @@ -524,12 +524,6 @@ func getFakeCluster(changes ...func(*clusterv1.Cluster)) *clusterv1.Cluster { InfrastructureReady: true, }, } - - for _, change := range changes { - change(input) - } - - return input } func getFakeAzureCluster(changes ...func(*infrav1.AzureCluster)) *infrav1.AzureCluster { @@ -537,13 +531,6 @@ func getFakeAzureCluster(changes ...func(*infrav1.AzureCluster)) *infrav1.AzureC ObjectMeta: metav1.ObjectMeta{ Name: "my-azure-cluster", Namespace: "default", - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "cluster.x-k8s.io/v1beta1", - Kind: "Cluster", - Name: "my-cluster", - }, - }, }, Spec: infrav1.AzureClusterSpec{ AzureClusterClassSpec: infrav1.AzureClusterClassSpec{ @@ -558,6 +545,20 @@ func getFakeAzureCluster(changes ...func(*infrav1.AzureCluster)) *infrav1.AzureC }, }, }, + APIServerLB: infrav1.LoadBalancerSpec{ + Name: "my-cluster-public-lb", + FrontendIPs: []infrav1.FrontendIP{ + { + PublicIP: &infrav1.PublicIPSpec{ + Name: "my-cluster-public-lb-frontEnd", + DNSName: "my-cluster-fb560e20.westus2.cloudapp.azure.com", + }, + }, + }, + }, + }, + ControlPlaneEndpoint: clusterv1.APIEndpoint{ + Port: 6443, }, }, }