Skip to content

Commit

Permalink
Merge pull request #3783 from nojnhuh/aso-pause-azuremanagedcontrolplane
Browse files Browse the repository at this point in the history
add pause handling for AzureManagedControlPlane
  • Loading branch information
k8s-ci-robot authored Aug 4, 2023
2 parents a617ebd + 150f44f commit 3b81e27
Show file tree
Hide file tree
Showing 4 changed files with 227 additions and 24 deletions.
61 changes: 37 additions & 24 deletions controllers/azuremanagedcontrolplane_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ func (amcpr *AzureManagedControlPlaneReconciler) SetupWithManager(ctx context.Co
c, err := ctrl.NewControllerManagedBy(mgr).
WithOptions(options.Options).
For(azManagedControlPlane).
WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(log, amcpr.WatchFilterValue)).
WithEventFilter(predicates.ResourceHasFilterLabel(log, amcpr.WatchFilterValue)).
// watch AzureManagedCluster resources
Watches(
&source.Kind{Type: &infrav1.AzureManagedCluster{}},
Expand All @@ -95,12 +95,12 @@ func (amcpr *AzureManagedControlPlaneReconciler) SetupWithManager(ctx context.Co
return errors.Wrap(err, "error creating controller")
}

// Add a watch on clusterv1.Cluster object for unpause & ready notifications.
// Add a watch on clusterv1.Cluster object for pause/unpause & ready notifications.
if err = c.Watch(
&source.Kind{Type: &clusterv1.Cluster{}},
handler.EnqueueRequestsFromMapFunc(amcpr.ClusterToAzureManagedControlPlane),
predicates.ClusterUnpausedAndInfrastructureReady(log),
predicates.ResourceNotPausedAndHasFilterLabel(log, amcpr.WatchFilterValue),
predicates.Any(log, predicates.ClusterCreateInfraReady(log), predicates.ClusterUpdateInfraReady(log), ClusterUpdatePauseChange(log)),
predicates.ResourceHasFilterLabel(log, amcpr.WatchFilterValue),
); err != nil {
return errors.Wrap(err, "failed adding a watch for ready clusters")
}
Expand Down Expand Up @@ -146,12 +146,6 @@ func (amcpr *AzureManagedControlPlaneReconciler) Reconcile(ctx context.Context,

log = log.WithValues("cluster", cluster.Name)

// Return early if the object or Cluster is paused.
if annotations.IsPaused(cluster, azureControlPlane) {
log.Info("AzureManagedControlPlane or linked Cluster is marked as paused. Won't reconcile")
return ctrl.Result{}, nil
}

// Fetch all the ManagedMachinePools owned by this Cluster.
opt1 := client.InNamespace(azureControlPlane.Namespace)
opt2 := client.MatchingLabels(map[string]string{
Expand All @@ -177,20 +171,6 @@ func (amcpr *AzureManagedControlPlaneReconciler) Reconcile(ctx context.Context,
}
}

// check if the control plane's namespace is allowed for this identity and update owner references for the identity.
if azureControlPlane.Spec.IdentityRef != nil {
err := EnsureClusterIdentity(ctx, amcpr.Client, azureControlPlane, azureControlPlane.Spec.IdentityRef, infrav1.ManagedClusterFinalizer)
if err != nil {
return reconcile.Result{}, err
}
} else {
warningMessage := ("You're using deprecated functionality: ")
warningMessage += ("Using Azure credentials from the manager environment is deprecated and will be removed in future releases. ")
warningMessage += ("Please specify an AzureClusterIdentity for the AzureManagedControlPlane instead, see: https://capz.sigs.k8s.io/topics/multitenancy.html ")
log.Info(fmt.Sprintf("WARNING, %s", warningMessage))
amcpr.Recorder.Eventf(azureControlPlane, corev1.EventTypeWarning, "AzureClusterIdentity", warningMessage)
}

// Create the scope.
mcpScope, err := scope.NewManagedControlPlaneScope(ctx, scope.ManagedControlPlaneScopeParams{
Client: amcpr.Client,
Expand All @@ -209,6 +189,26 @@ func (amcpr *AzureManagedControlPlaneReconciler) Reconcile(ctx context.Context,
}
}()

// Return early if the object or Cluster is paused.
if annotations.IsPaused(cluster, azureControlPlane) {
log.Info("AzureManagedControlPlane or linked Cluster is marked as paused. Won't reconcile normally")
return amcpr.reconcilePause(ctx, mcpScope)
}

// check if the control plane's namespace is allowed for this identity and update owner references for the identity.
if azureControlPlane.Spec.IdentityRef != nil {
err := EnsureClusterIdentity(ctx, amcpr.Client, azureControlPlane, azureControlPlane.Spec.IdentityRef, infrav1.ManagedClusterFinalizer)
if err != nil {
return reconcile.Result{}, err
}
} else {
warningMessage := "You're using deprecated functionality: " +
"Using Azure credentials from the manager environment is deprecated and will be removed in future releases. " +
"Please specify an AzureClusterIdentity for the AzureManagedControlPlane instead, see: https://capz.sigs.k8s.io/topics/multitenancy.html "
log.Info(fmt.Sprintf("WARNING, %s", warningMessage))
amcpr.Recorder.Eventf(azureControlPlane, corev1.EventTypeWarning, "AzureClusterIdentity", warningMessage)
}

// Handle deleted clusters
if !azureControlPlane.DeletionTimestamp.IsZero() {
return amcpr.reconcileDelete(ctx, mcpScope)
Expand Down Expand Up @@ -263,6 +263,19 @@ func (amcpr *AzureManagedControlPlaneReconciler) reconcileNormal(ctx context.Con
return reconcile.Result{}, nil
}

func (amcpr *AzureManagedControlPlaneReconciler) reconcilePause(ctx context.Context, scope *scope.ManagedControlPlaneScope) (reconcile.Result, error) {
ctx, log, done := tele.StartSpanWithLogger(ctx, "controllers.AzureManagedControlPlane.reconcilePause")
defer done()

log.Info("Reconciling AzureManagedControlPlane pause")

if err := newAzureManagedControlPlaneReconciler(scope).Pause(ctx); err != nil {
return reconcile.Result{}, errors.Wrap(err, "failed to pause control plane services")
}

return reconcile.Result{}, nil
}

func (amcpr *AzureManagedControlPlaneReconciler) reconcileDelete(ctx context.Context, scope *scope.ManagedControlPlaneScope) (reconcile.Result, error) {
ctx, log, done := tele.StartSpanWithLogger(ctx, "controllers.AzureManagedControlPlaneReconciler.reconcileDelete")
defer done()
Expand Down
74 changes: 74 additions & 0 deletions controllers/azuremanagedcontrolplane_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,22 @@ limitations under the License.
package controllers

import (
"context"
"testing"

. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
"sigs.k8s.io/cluster-api-provider-azure/internal/test"
"sigs.k8s.io/cluster-api-provider-azure/util/reconciler"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
)

func TestClusterToAzureManagedControlPlane(t *testing.T) {
Expand Down Expand Up @@ -77,3 +86,68 @@ func TestClusterToAzureManagedControlPlane(t *testing.T) {
})
}
}

func TestAzureManagedControlPlaneReconcilePaused(t *testing.T) {
g := NewWithT(t)

ctx := context.Background()

sb := runtime.NewSchemeBuilder(
clusterv1.AddToScheme,
infrav1.AddToScheme,
)
s := runtime.NewScheme()
g.Expect(sb.AddToScheme(s)).To(Succeed())
c := fake.NewClientBuilder().
WithScheme(s).
Build()

recorder := record.NewFakeRecorder(1)

reconciler := &AzureManagedControlPlaneReconciler{
Client: c,
Recorder: recorder,
ReconcileTimeout: reconciler.DefaultLoopTimeout,
WatchFilterValue: "",
}
name := test.RandomName("paused", 10)

cluster := &clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: "default",
},
Spec: clusterv1.ClusterSpec{
Paused: true,
},
}
g.Expect(c.Create(ctx, cluster)).To(Succeed())

instance := &infrav1.AzureManagedControlPlane{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: "default",
OwnerReferences: []metav1.OwnerReference{
{
Kind: "Cluster",
APIVersion: clusterv1.GroupVersion.String(),
Name: cluster.Name,
},
},
},
Spec: infrav1.AzureManagedControlPlaneSpec{
SubscriptionID: "something",
},
}
g.Expect(c.Create(ctx, instance)).To(Succeed())

result, err := reconciler.Reconcile(context.Background(), ctrl.Request{
NamespacedName: client.ObjectKey{
Namespace: instance.Namespace,
Name: instance.Name,
},
})

g.Expect(err).To(BeNil())
g.Expect(result.RequeueAfter).To(BeZero())
}
18 changes: 18 additions & 0 deletions controllers/azuremanagedcontrolplane_reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,24 @@ func (r *azureManagedControlPlaneService) Reconcile(ctx context.Context) error {
return nil
}

// Pause pauses all components making up the cluster.
func (r *azureManagedControlPlaneService) Pause(ctx context.Context) error {
ctx, _, done := tele.StartSpanWithLogger(ctx, "controllers.azureManagedControlPlaneService.Pause")
defer done()

for _, service := range r.services {
pauser, ok := service.(azure.Pauser)
if !ok {
continue
}
if err := pauser.Pause(ctx); err != nil {
return errors.Wrapf(err, "failed to pause AzureManagedControlPlane service %s", service.Name())
}
}

return nil
}

// Delete reconciles all the services in a predetermined order.
func (r *azureManagedControlPlaneService) Delete(ctx context.Context) error {
ctx, _, done := tele.StartSpanWithLogger(ctx, "controllers.azureManagedControlPlaneService.Delete")
Expand Down
98 changes: 98 additions & 0 deletions controllers/azuremanagedcontrolplane_reconciler_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controllers

import (
"context"
"testing"

. "github.com/onsi/gomega"
"github.com/pkg/errors"
"go.uber.org/mock/gomock"
"sigs.k8s.io/cluster-api-provider-azure/azure"
"sigs.k8s.io/cluster-api-provider-azure/azure/mock_azure"
gomockinternal "sigs.k8s.io/cluster-api-provider-azure/internal/test/matchers/gomock"
)

func TestAzureManagedControlPlaneServicePause(t *testing.T) {
type pausingServiceReconciler struct {
*mock_azure.MockServiceReconciler
*mock_azure.MockPauser
}

cases := map[string]struct {
expectedError string
expect func(one pausingServiceReconciler, two pausingServiceReconciler, three pausingServiceReconciler)
}{
"all services are paused in order": {
expectedError: "",
expect: func(one pausingServiceReconciler, two pausingServiceReconciler, three pausingServiceReconciler) {
gomock.InOrder(
one.MockPauser.EXPECT().Pause(gomockinternal.AContext()).Return(nil),
two.MockPauser.EXPECT().Pause(gomockinternal.AContext()).Return(nil),
three.MockPauser.EXPECT().Pause(gomockinternal.AContext()).Return(nil))
},
},
"service pause fails": {
expectedError: "failed to pause AzureManagedControlPlane service two: some error happened",
expect: func(one pausingServiceReconciler, two pausingServiceReconciler, _ pausingServiceReconciler) {
gomock.InOrder(
one.MockPauser.EXPECT().Pause(gomockinternal.AContext()).Return(nil),
two.MockPauser.EXPECT().Pause(gomockinternal.AContext()).Return(errors.New("some error happened")),
two.MockServiceReconciler.EXPECT().Name().Return("two"))
},
},
}

for name, tc := range cases {
tc := tc
t.Run(name, func(t *testing.T) {
g := NewWithT(t)

t.Parallel()
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()

newPausingServiceReconciler := func() pausingServiceReconciler {
return pausingServiceReconciler{
mock_azure.NewMockServiceReconciler(mockCtrl),
mock_azure.NewMockPauser(mockCtrl),
}
}
svcOneMock := newPausingServiceReconciler()
svcTwoMock := newPausingServiceReconciler()
svcThreeMock := newPausingServiceReconciler()

tc.expect(svcOneMock, svcTwoMock, svcThreeMock)

s := &azureManagedControlPlaneService{
services: []azure.ServiceReconciler{
svcOneMock,
svcTwoMock,
svcThreeMock,
},
}

err := s.Pause(context.TODO())
if tc.expectedError != "" {
g.Expect(err).To(HaveOccurred())
g.Expect(err).To(MatchError(tc.expectedError))
} else {
g.Expect(err).NotTo(HaveOccurred())
}
})
}
}

0 comments on commit 3b81e27

Please sign in to comment.