Skip to content

Commit

Permalink
Merge pull request #2582 from hankfreund/delete_annotations
Browse files Browse the repository at this point in the history 
Add support for deleting resources in reverse order by stage.
  • Loading branch information
@google-oss-prow
google-oss-prow[bot] authored Aug 29, 2024
2 parents f402263 + 96ea0c1 commit 0784071
Show file tree
Hide file tree
Showing 8 changed files with 712 additions and 253 deletions.
182 changes: 182 additions & 0 deletions experiments/compositions/composition/internal/controller/expander_reconciler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,15 @@ import (
"golang.org/x/time/rate"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/selection"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/rest"
Expand All @@ -44,11 +47,13 @@ import (
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/source"
"sigs.k8s.io/kubebuilder-declarative-pattern/applylib/forked/github.com/kubernetes/kubectl/pkg/cmd/apply"
)

// ExpanderReconciler reconciles a expander object
Expand All @@ -69,6 +74,11 @@ type EvaluateWaitError struct {
msg string
}

const (
finalizerName = "compositions.google.com/expander-finalizer"
stagesAnnotation = "compositions.google.com/expander-stages"
)

func (e *EvaluateWaitError) Error() string { return e.msg }

var contextGVK schema.GroupVersionKind = schema.GroupVersionKind{
Expand Down Expand Up @@ -138,6 +148,16 @@ func (r *ExpanderReconciler) getPlanForInputCR(ctx context.Context, inputcr *uns
return planNN, &plancr, err
}
}

// Add a finalizer to the Plan to discourage out of band deletions. Stage information is
// stored in the Plan to handle proper cleanup of resources when a Facade is deleted.
if !controllerutil.ContainsFinalizer(&plancr, finalizerName) {
controllerutil.AddFinalizer(&plancr, finalizerName)
if err := r.Update(ctx, &plancr); err != nil {
logger.Error(err, "Unable to add finalizer to Plan Object")
return planNN, &plancr, err
}
}
return planNN, &plancr, nil
}

Expand All @@ -158,6 +178,18 @@ func (r *ExpanderReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
return ctrl.Result{}, client.IgnoreNotFound(err)
}

if !inputcr.GetDeletionTimestamp().IsZero() {
return r.reconcileDelete(ctx, logger, inputcr)
}
// Add a finalizer to prevent removal of facade before all applied objects are cleaned up.
if !controllerutil.ContainsFinalizer(&inputcr, finalizerName) {
controllerutil.AddFinalizer(&inputcr, finalizerName)
if err := r.Update(ctx, &inputcr); err != nil {
logger.Error(err, "Unable to add finalizer to input CR")
return ctrl.Result{}, err
}
}

// Grab the latest composition
// TODO(barni@) - Decide how we want the latest composition changes are to be applied.
var compositionCR compositionv1alpha1.Composition
Expand Down Expand Up @@ -266,6 +298,15 @@ func (r *ExpanderReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
logger.Error(err, "unable to read Plan CR")
return ctrl.Result{}, err
}

// Write out the (in-order) stages to the plan as a reference for later when we need to delete resources.
metav1.SetMetaDataAnnotation(&plancr.ObjectMeta, stagesAnnotation, strings.Join(stagesEvaluated, ","))
err = r.Client.Update(ctx, plancr)
if err != nil {
logger.Error(err, "error updating plan stages", "plan", planNN)
return ctrl.Result{}, err
}

if planUpdated && oldGeneration == plancr.GetGeneration() {
logger.Info("Did not get the latest Plan CR. Will retry.", "generation", oldGeneration)
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
Expand Down Expand Up @@ -665,6 +706,147 @@ func (r *ExpanderReconciler) enqueueAllFromGVK(ctx context.Context, _ client.Obj
return reqs
}

func (r *ExpanderReconciler) reconcileDelete(ctx context.Context, logger logr.Logger, inputcr unstructured.Unstructured) (ctrl.Result, error) {
logger = logger.WithName("Delete")
if !controllerutil.ContainsFinalizer(&inputcr, finalizerName) {
return ctrl.Result{}, nil
}
planNN := types.NamespacedName{
Name: r.InputGVR.Resource + "-" + inputcr.GetName(),
Namespace: inputcr.GetNamespace(),
}
plancr := compositionv1alpha1.Plan{}
if err := r.Client.Get(ctx, planNN, &plancr); err != nil {
logger.Error(err, "Unable to fetch Plan Object", "plan", planNN)
return ctrl.Result{}, err
}
annotations := plancr.GetAnnotations()
planLabels := plancr.GetLabels()
stageList, ok := annotations[stagesAnnotation]
if !ok {
err := fmt.Errorf("Plan is missing stage order annotation")
logger.Error(err, "Unable to fetch stage order from Plan", "Plan", planNN)
return ctrl.Result{}, err
}
stages := strings.Split(stageList, ",")
numFound := 0
for i := len(stages) - 1; i >= 0; i-- {
r.Recorder.Eventf(&inputcr, corev1.EventTypeNormal, "Delete", "Deleting objects for stage %s", stages[i])
nsList, ok := annotations[apply.ApplySetAdditionalNamespacesAnnotation]
if !ok {
err := fmt.Errorf("Plan is missing Namespace annotation")
logger.Error(err, "Unable to fetch Namespaces from Plan", "Plan", planNN)
return ctrl.Result{}, err
}
namespaces := strings.Split(nsList, ",")
namespaces = append(namespaces, inputcr.GetNamespace())
opts, err := deleteListOpts(stages[i], planLabels[apply.ApplySetParentIDLabel])
if err != nil {
logger.Error(err, "Error creating list options")
return ctrl.Result{}, err
}
gkList, ok := annotations[apply.ApplySetGKsAnnotation]
if !ok {
err := fmt.Errorf("Plan is missing GroupKind annotation")
logger.Error(err, "Unable to fetch GroupKinds from Plan", "Plan", planNN)
return ctrl.Result{}, err
}
gks := strings.Split(gkList, ",")
for _, gk := range gks {
parsedGK := schema.ParseGroupKind(gk)
mapping, err := r.RESTMapper.RESTMapping(parsedGK)
if err != nil {
return ctrl.Result{}, err
}
n := 0
if mapping.Scope.Name() == meta.RESTScopeNameNamespace {
n, err = r.deleteNamespacedResources(ctx, logger, stages[i], mapping.Resource, namespaces, opts)
} else if mapping.Scope.Name() == meta.RESTScopeNameRoot {
n, err = r.deleteClusterResources(ctx, logger, stages[i], mapping.Resource, opts)
}
if err != nil {
logger.Error(err, "Error deleting resources", "GroupKind", gk)
r.Recorder.Eventf(&inputcr, corev1.EventTypeWarning, "Delete", "Failed deleting objects of GroupKind %q for stage %q: %v", gk, stages[i], err)
return ctrl.Result{}, err
}
numFound += n
}
if numFound > 0 {
break
}
}
if numFound > 0 {
return ctrl.Result{Requeue: true, RequeueAfter: 10 * time.Second}, nil
}

// Remove the finalizers to allow the Plan and Facade to be deleted.
controllerutil.RemoveFinalizer(&plancr, finalizerName)
if err := r.Update(ctx, &plancr); err != nil {
logger.Error(err, "Unable to remove finalizer from Plan")
return ctrl.Result{}, err
}
controllerutil.RemoveFinalizer(&inputcr, finalizerName)
if err := r.Update(ctx, &inputcr); err != nil {
logger.Error(err, "Unable to remove finalizer from input CR")
return ctrl.Result{}, err
}
// Expanded resources are all deleted, stop reconciliation.
return ctrl.Result{}, nil
}

func deleteListOpts(stage, applysetId string) (metav1.ListOptions, error) {
stageReq, err := labels.NewRequirement(applier.StageLabel, selection.Equals, []string{stage})
if err != nil {
return metav1.ListOptions{}, fmt.Errorf("failed making stage label selector: %v", err)
}
asReq, err := labels.NewRequirement(apply.ApplysetPartOfLabel, selection.Equals, []string{applysetId})
if err != nil {
return metav1.ListOptions{}, fmt.Errorf("failed making applysetId label selector: %v", err)
}
opts := metav1.ListOptions{
LabelSelector: labels.NewSelector().Add(*stageReq).Add(*asReq).String(),
}
return opts, nil
}

func (r *ExpanderReconciler) deleteNamespacedResources(ctx context.Context, logger logr.Logger, stage string, endpoint schema.GroupVersionResource, namespaces []string, opts metav1.ListOptions) (int, error) {
numFound := 0
for _, ns := range namespaces {
resources, err := r.Dynamic.Resource(endpoint).Namespace(ns).List(ctx, opts)
if err != nil {
return numFound, fmt.Errorf("error listing resources in Namespace %q for stage %q: %v", ns, stage, err)
}
for _, res := range resources.Items {
logger.Info("Attempting to delete resource", "Resource", res, "Namespace", ns)
err := r.Delete(ctx, &res)
if err == nil {
numFound++
} else if err != nil && !apierrors.IsNotFound(err) {
return numFound, fmt.Errorf("failed deleting object %v in Namespace %q for stage %q: %v", res, ns, stage, err)
}
}
}
return numFound, nil
}

func (r *ExpanderReconciler) deleteClusterResources(ctx context.Context, logger logr.Logger, stage string, endpoint schema.GroupVersionResource, opts metav1.ListOptions) (int, error) {
numFound := 0
resources, err := r.Dynamic.Resource(endpoint).List(ctx, opts)
if err != nil {
return numFound, fmt.Errorf("error listing resources for stage %q: %v", stage, err)
}
for _, res := range resources.Items {
logger.Info("Attempting to delete resource", "Resource", res)
err := r.Delete(ctx, &res)
if err == nil {
numFound++
} else if err != nil && !apierrors.IsNotFound(err) {
return numFound, fmt.Errorf("failed deleting object %v for stage %q: %v", res, stage, err)
}
}
return numFound, nil
}

// SetupWithManager sets up the controller with the Manager.
func (r *ExpanderReconciler) SetupWithManager(mgr ctrl.Manager, cr *unstructured.Unstructured) error {
var err error
Expand Down
10 changes: 10 additions & 0 deletions experiments/compositions/composition/pkg/applier/applier.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ import (
"sigs.k8s.io/kubebuilder-declarative-pattern/pkg/patterns/declarative/pkg/manifest"
)

const StageLabel = "compositions.google.com/applier-stage"

type ApplierClient struct {
RESTMapper meta.RESTMapper
Dynamic *dynamic.DynamicClient
Expand Down Expand Up @@ -169,6 +171,7 @@ func (a *Applier) Load() error {
a.logger.Error(err, "Error injecting ownerRefs")
return err
}
a.addStageLabel(objects)

a.objects = []applyset.ApplyableObject{}
// loop over objects and extract unstructured
Expand Down Expand Up @@ -216,6 +219,13 @@ func (a *Applier) injectOwnerRef(objects *manifest.Objects) error {
return nil
}

func (a *Applier) addStageLabel(objects *manifest.Objects) {
labels := map[string]string{StageLabel: a.stageName}
for _, o := range objects.Items {
o.AddLabels(labels)
}
}

func (a *Applier) getApplyOptions(prune bool) (applyset.Options, error) {
var options applyset.Options
force := true
Expand Down
Loading

0 comments on commit 0784071

Please sign in to comment.