Skip to content

Commit

Permalink
feat: automatic changes to workloads when an AuthProxyWorload is dele…
Browse files Browse the repository at this point in the history 
…ted (#200)

When an AuthProxyWorkload is deleted, automatically update all related workloads that support automatic
rollout: Deployment, StatefulSet, DaemonSet, ReplicaSet. 

Related to #187
  • Loading branch information
@hessjcg
hessjcg authored Feb 8, 2023
1 parent c523cd0 commit e11caed
Show file tree
Hide file tree
Showing 8 changed files with 311 additions and 93 deletions.
64 changes: 36 additions & 28 deletions internal/controller/authproxyworkload_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ func (r *AuthProxyWorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Re
"gen", resource.GetGeneration())
r.recentlyDeleted.set(req.NamespacedName, true)
// the object has been deleted
return r.doDelete(ctx, resource, l)
return r.doDelete(ctx, resource)
}

l.Info("Reconcile add/update for AuthProxyWorkload",
Expand All @@ -169,10 +169,15 @@ func (r *AuthProxyWorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Re

// doDelete removes our finalizer and updates the related workloads
// when the reconcile loop receives an AuthProxyWorkload that was deleted.
func (r *AuthProxyWorkloadReconciler) doDelete(ctx context.Context, resource *cloudsqlapi.AuthProxyWorkload, l logr.Logger) (ctrl.Result, error) {
func (r *AuthProxyWorkloadReconciler) doDelete(ctx context.Context, resource *cloudsqlapi.AuthProxyWorkload) (ctrl.Result, error) {

// Mark all related workloads as needing to be updated
_, err := r.updateWorkloadStatus(ctx, l, resource)
allWorkloads, err := r.updateWorkloadStatus(ctx, resource)
if err != nil {
return requeueNow, err
}

_, err = r.updateWorkloadAnnotations(ctx, resource, allWorkloads)
if err != nil {
return requeueNow, err
}
Expand Down Expand Up @@ -231,7 +236,7 @@ func (r *AuthProxyWorkloadReconciler) doCreateUpdate(ctx context.Context, l logr
}

// find all workloads that relate to this AuthProxyWorkload resource
allWorkloads, err := r.updateWorkloadStatus(ctx, l, resource)
allWorkloads, err := r.updateWorkloadStatus(ctx, resource)
if err != nil {
// State 1.2 - unable to read workloads, abort and try again after a delay.
return requeueWithDelay, err
Expand All @@ -247,25 +252,9 @@ func (r *AuthProxyWorkloadReconciler) doCreateUpdate(ctx context.Context, l logr

// State 3.*: Workloads already exist. Some may need to be updated to roll out
// changes.
var outOfDateCount int
for _, wl := range allWorkloads {
wlChanged := r.needsAnnotationUpdate(wl, resource)
if !wlChanged {
continue
}

outOfDateCount++
_, err = controllerutil.CreateOrPatch(ctx, r.Client, wl.Object(), func() error {
r.updateAnnotation(wl, resource)
return nil
})

// State 3.1 Failed to update one of the workloads PodTemplateSpec annotations, requeue.
if err != nil {
message := fmt.Sprintf("Reconciled %d matching workloads. Error updating workload %v: %v", len(allWorkloads), wl.Object().GetName(), err)
return r.reconcileResult(ctx, l, resource, orig, cloudsqlapi.ReasonWorkloadNeedsUpdate, message, false)
}

outOfDateCount, err := r.updateWorkloadAnnotations(ctx, resource, allWorkloads)
if err != nil {
return requeueNow, err
}

// State 3.2 Successfully updated all workload PodTemplateSpec annotations, requeue
Expand All @@ -282,14 +271,12 @@ func (r *AuthProxyWorkloadReconciler) doCreateUpdate(ctx context.Context, l logr
// needsAnnotationUpdate returns true when the workload was annotated with
// a different generation of the resource.
func (r *AuthProxyWorkloadReconciler) needsAnnotationUpdate(wl workload.Workload, resource *cloudsqlapi.AuthProxyWorkload) bool {

// This workload is not mutable. Ignore it.
if _, ok := wl.(workload.WithMutablePodTemplate); !ok {
return false
}

k, v := workload.PodAnnotation(resource)

// Check if the correct annotation exists
an := wl.PodTemplateAnnotations()
if an != nil && an[k] == v {
Expand Down Expand Up @@ -388,16 +375,15 @@ func (r *AuthProxyWorkloadReconciler) patchAuthProxyWorkloadStatus(
// updates the needs update annotations using internal.UpdateWorkloadAnnotation.
// Once the workload is saved, the workload admission mutate webhook will
// apply the correct containers to this instance.
func (r *AuthProxyWorkloadReconciler) updateWorkloadStatus(ctx context.Context, _ logr.Logger, resource *cloudsqlapi.AuthProxyWorkload) (matching []workload.Workload, retErr error) {
func (r *AuthProxyWorkloadReconciler) updateWorkloadStatus(ctx context.Context, resource *cloudsqlapi.AuthProxyWorkload) (matching []workload.Workload, retErr error) {

matching, err := r.listWorkloads(ctx, resource.Spec.Workload, resource.GetNamespace())
if err != nil {
return nil, err
}

// all matching workloads get a new annotation that will be removed
// when the reconcile loop for outOfDate is completed.
for _, wl := range matching {
// update the status condition for a workload
s := newStatus(wl)
s.Conditions = replaceCondition(s.Conditions, &metav1.Condition{
Type: cloudsqlapi.ConditionWorkloadUpToDate,
Expand Down Expand Up @@ -531,3 +517,25 @@ func (r *AuthProxyWorkloadReconciler) loadByLabelSelector(ctx context.Context, w
return wl.Workloads(), nil

}

func (r *AuthProxyWorkloadReconciler) updateWorkloadAnnotations(ctx context.Context, resource *cloudsqlapi.AuthProxyWorkload, workloads []workload.Workload) (int, error) {
var outOfDate int
for _, wl := range workloads {
if r.needsAnnotationUpdate(wl, resource) {
outOfDate++

_, err := controllerutil.CreateOrPatch(ctx, r.Client, wl.Object(), func() error {
r.updateAnnotation(wl, resource)
return nil
})

// Failed to update one of the workloads PodTemplateSpec annotations.
if err != nil {
return 0, fmt.Errorf("reconciled %d matching workloads. Error removing proxy from workload %v: %v", len(workloads), wl.Object().GetName(), err)
}
}
}

return outOfDate, nil

}
175 changes: 125 additions & 50 deletions internal/controller/authproxyworkload_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"context"
"fmt"
"os"
"strings"
"testing"

"go.uber.org/zap/zapcore"
Expand Down Expand Up @@ -69,11 +70,8 @@ func TestReconcileDeleted(t *testing.T) {
Namespace: "default",
Name: "test",
}, "project:region:db")
p.Finalizers = []string{finalizerName}
p.Spec.Workload = v1alpha1.WorkloadSelectorSpec{
Kind: "Pod",
Name: "thing",
}
addFinalizers(p)
addPodWorkload(p)

cb, err := clientBuilder()
if err != nil {
Expand Down Expand Up @@ -114,11 +112,8 @@ func TestReconcileState21ByName(t *testing.T) {
Namespace: "default",
Name: "test",
}, "project:region:db")
p.Finalizers = []string{finalizerName}
p.Spec.Workload = v1alpha1.WorkloadSelectorSpec{
Kind: "Pod",
Name: "testpod",
}
addFinalizers(p)
addPodWorkload(p)

err := runReconcileTestcase(p, []client.Object{p}, false, metav1.ConditionTrue, v1alpha1.ReasonNoWorkloadsFound)
if err != nil {
Expand All @@ -131,13 +126,8 @@ func TestReconcileState21BySelector(t *testing.T) {
Namespace: "default",
Name: "test",
}, "project:region:db")
p.Finalizers = []string{finalizerName}
p.Spec.Workload = v1alpha1.WorkloadSelectorSpec{
Kind: "Pod",
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"app": "things"},
},
}
addFinalizers(p)
addSelectorWorkload(p, "Pod", "app", "things")

err := runReconcileTestcase(p, []client.Object{p}, false, metav1.ConditionTrue, v1alpha1.ReasonNoWorkloadsFound)
if err != nil {
Expand All @@ -147,35 +137,28 @@ func TestReconcileState21BySelector(t *testing.T) {
}

func TestReconcileState32(t *testing.T) {
wantRequeue := true
wantStatus := metav1.ConditionFalse
wantReason := v1alpha1.ReasonWorkloadNeedsUpdate

const (
wantRequeue = true
wantStatus = metav1.ConditionFalse
wantReason = v1alpha1.ReasonWorkloadNeedsUpdate
labelK = "app"
labelV = "things"
)
p := testhelpers.BuildAuthProxyWorkload(types.NamespacedName{
Namespace: "default",
Name: "test",
}, "project:region:db")
p.Generation = 2
p.Finalizers = []string{finalizerName}
p.Spec.Workload = v1alpha1.WorkloadSelectorSpec{
Kind: "Deployment",
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"app": "things"},
},
}
p.Status.Conditions = []*metav1.Condition{{
Type: v1alpha1.ConditionUpToDate,
Reason: v1alpha1.ReasonStartedReconcile,
Status: metav1.ConditionFalse,
}}
addFinalizers(p)
addSelectorWorkload(p, "Deployment", labelK, labelV)

// mimic a pod that was updated by the webhook
reqName := v1alpha1.AnnotationPrefix + "/" + p.Name
pod := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "thing",
Namespace: "default",
Labels: map[string]string{"app": "things"},
Labels: map[string]string{labelK: labelV},
},
Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{reqName: "1"}},
Expand All @@ -190,35 +173,29 @@ func TestReconcileState32(t *testing.T) {
}

func TestReconcileState33(t *testing.T) {
wantRequeue := false
wantStatus := metav1.ConditionTrue
wantReason := v1alpha1.ReasonFinishedReconcile
const (
wantRequeue = false
wantStatus = metav1.ConditionTrue
wantReason = v1alpha1.ReasonFinishedReconcile
labelK = "app"
labelV = "things"
)

p := testhelpers.BuildAuthProxyWorkload(types.NamespacedName{
Namespace: "default",
Name: "test",
}, "project:region:db")
p.Generation = 1
p.Finalizers = []string{finalizerName}
p.Spec.Workload = v1alpha1.WorkloadSelectorSpec{
Kind: "Deployment",
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"app": "things"},
},
}
p.Status.Conditions = []*metav1.Condition{{
Type: v1alpha1.ConditionUpToDate,
Reason: v1alpha1.ReasonStartedReconcile,
Status: metav1.ConditionFalse,
}}
addFinalizers(p)
addSelectorWorkload(p, "Deployment", labelK, labelV)

// mimic a pod that was updated by the webhook
reqName := v1alpha1.AnnotationPrefix + "/" + p.Name
pod := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "thing",
Namespace: "default",
Labels: map[string]string{"app": "things"},
Labels: map[string]string{labelK: labelV},
},
Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{reqName: "1"}},
Expand All @@ -232,6 +209,86 @@ func TestReconcileState33(t *testing.T) {

}

func TestReconcileDeleteUpdatesWorkload(t *testing.T) {
const (
labelK = "app"
labelV = "things"
)
resource := testhelpers.BuildAuthProxyWorkload(types.NamespacedName{
Namespace: "default",
Name: "test",
}, "project:region:db")
resource.Generation = 1
addFinalizers(resource)
addSelectorWorkload(resource, "Deployment", labelK, labelV)

k, v := workload.PodAnnotation(resource)

// mimic a deployment that was updated by the webhook
deployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "thing",
Namespace: "default",
Labels: map[string]string{labelK: labelV},
},
Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{k: v}},
}},
}

// Build a client with the resource and deployment
cb, err := clientBuilder()
if err != nil {
t.Error(err) // shouldn't ever happen
}
c := cb.WithObjects(resource, deployment).Build()
r, req, ctx := reconciler(resource, c)

// Delete the resource
c.Delete(ctx, resource)
if err != nil {
t.Error(err)
}

// Run Reconcile on the deleted resource
res, err := r.Reconcile(ctx, req)
if err != nil {
t.Error(err)
}
if res.Requeue {
t.Errorf("got %v, want %v for requeue", res.Requeue, false)
}

// Check that the resource doesn't exist anymore
err = c.Get(ctx, types.NamespacedName{
Namespace: resource.GetNamespace(),
Name: resource.GetName(),
}, resource)
if err != nil {
if !errors.IsNotFound(err) {
t.Errorf("wants not found error, got %v", err)
}
} else {
t.Error("wants not found error, got no error")
}

// Fetch the deployment and make sure the annotations show the
// deleted resource.
d := &appsv1.Deployment{}
err = c.Get(ctx, types.NamespacedName{
Namespace: deployment.GetNamespace(),
Name: deployment.GetName(),
}, d)
if err != nil {
t.Fatal(err)
}

if got, want := d.Spec.Template.ObjectMeta.Annotations[k], "1-deleted-"; !strings.HasPrefix(got, want) {
t.Fatalf("got %v, wants annotation value to have prefix %v", got, want)
}

}

func runReconcileTestcase(p *v1alpha1.AuthProxyWorkload, clientObjects []client.Object, wantRequeue bool, wantStatus metav1.ConditionStatus, wantReason string) error {
cb, err := clientBuilder()
if err != nil {
Expand Down Expand Up @@ -304,3 +361,21 @@ func reconciler(p *v1alpha1.AuthProxyWorkload, cb client.Client) (*AuthProxyWork
}
return r, req, ctx
}

func addFinalizers(p *v1alpha1.AuthProxyWorkload) {
p.Finalizers = []string{finalizerName}
}
func addPodWorkload(p *v1alpha1.AuthProxyWorkload) {
p.Spec.Workload = v1alpha1.WorkloadSelectorSpec{
Kind: "Pod",
Name: "testpod",
}
}
func addSelectorWorkload(p *v1alpha1.AuthProxyWorkload, kind, labelK, labelV string) {
p.Spec.Workload = v1alpha1.WorkloadSelectorSpec{
Kind: kind,
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{labelK: labelV},
},
}
}
6 changes: 3 additions & 3 deletions internal/testhelpers/resources.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -595,7 +595,7 @@ func BuildAuthProxyWorkload(key types.NamespacedName, connectionString string) *
}

// CreateAuthProxyWorkload creates an AuthProxyWorkload in the kubernetes cluster.
func (cc *TestCaseClient) CreateAuthProxyWorkload(ctx context.Context, key types.NamespacedName, appLabel string, connectionString string, kind string) error {
func (cc *TestCaseClient) CreateAuthProxyWorkload(ctx context.Context, key types.NamespacedName, appLabel string, connectionString string, kind string) (*v1alpha1.AuthProxyWorkload, error) {
proxy := BuildAuthProxyWorkload(key, connectionString)
proxy.Spec.Workload = v1alpha1.WorkloadSelectorSpec{
Kind: kind,
Expand All @@ -613,9 +613,9 @@ func (cc *TestCaseClient) CreateAuthProxyWorkload(ctx context.Context, key types
}
err := cc.Client.Create(ctx, proxy)
if err != nil {
return fmt.Errorf("Unable to create entity %v", err)
return nil, fmt.Errorf("Unable to create entity %v", err)
}
return nil
return proxy, nil
}

// GetConditionStatus finds a condition where Condition.Type == condType and returns
Expand Down
Loading

0 comments on commit e11caed

Please sign in to comment.