Add support for deleting resources in reverse order by stage.

This solution makes use of the ApplySet annotations for GroupKinds and
Namespaces. When a facade deletion request is received, the ApplySet
data is read from the Plan to know what GKs to find and delete, and
which namespaces to search. The resources are further filtered by the
ApplySet ID they are automatically labeled with.
Stage order information is added to the Plan so the deletion processing
can iterate over them in reverse.

experiments/compositions/composition/internal/controller/expander_reconciler.go

@@ -29,12 +29,15 @@ import (
 	"golang.org/x/time/rate"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
+	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/selection"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/rest"
@@ -44,6 +47,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
@@ -69,6 +73,15 @@ type EvaluateWaitError struct {
 	msg string
 }
 
+const (
+	finalizerName    = "compositions.google.com/expander-finalizer"
+	stagesAnnotation = "compositions.google.com/expander-stages"
+	nsAnnotation     = "applyset.kubernetes.io/additional-namespaces"
+	gkAnnotation     = "applyset.kubernetes.io/contains-group-kinds"
+	applysetIdLabel  = "applyset.kubernetes.io/id"
+	partOfLabel      = "applyset.kubernetes.io/part-of"
+)
+
 func (e *EvaluateWaitError) Error() string { return e.msg }
 
 var contextGVK schema.GroupVersionKind = schema.GroupVersionKind{
@@ -127,6 +140,9 @@ func (r *ExpanderReconciler) getPlanForInputCR(ctx context.Context, inputcr *uns
 				"name", plancr.Name, "namespace", plancr.Namespace)
 			return planNN, &plancr, err
 		}
+		// Add a finalizer to the Plan to discourage out of band deletions. Stage information is
+		// stored in the Plan to handle proper cleanup of resources when a Facade is deleted.
+		controllerutil.AddFinalizer(&plancr, finalizerName)
 		if err := r.Client.Create(ctx, &plancr); err != nil {
 			logger.Error(err, "Unable to create Plan Object")
 			return planNN, &plancr, err
@@ -158,6 +174,18 @@ func (r *ExpanderReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
+	if !inputcr.GetDeletionTimestamp().IsZero() {
+		return r.reconcileDelete(ctx, logger, inputcr)
+	}
+	// Add a finalizer to prevent removal of facade before all applied objects are cleaned up.
+	if !controllerutil.ContainsFinalizer(&inputcr, finalizerName) {
+		controllerutil.AddFinalizer(&inputcr, finalizerName)
+		if err := r.Update(ctx, &inputcr); err != nil {
+			logger.Error(err, "Unable to add finalizer to input CR")
+			return ctrl.Result{}, err
+		}
+	}
+
 	// Grab the latest composition
 	// TODO(barni@) - Decide how we want the latest composition changes are to be applied.
 	var compositionCR compositionv1alpha1.Composition
@@ -266,6 +294,15 @@ func (r *ExpanderReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 		logger.Error(err, "unable to read Plan CR")
 		return ctrl.Result{}, err
 	}
+
+	// Write out the (in-order) stages to the plan as a reference for later when we need to delete resources.
+	metav1.SetMetaDataAnnotation(&plancr.ObjectMeta, stagesAnnotation, strings.Join(stagesEvaluated, ","))
+	err = r.Client.Update(ctx, plancr)
+	if err != nil {
+		logger.Error(err, "error updating plan stages", "plan", planNN)
+		return ctrl.Result{}, err
+	}
+
 	if planUpdated && oldGeneration == plancr.GetGeneration() {
 		logger.Info("Did not get the latest Plan CR. Will retry.", "generation", oldGeneration)
 		return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
@@ -665,6 +702,147 @@ func (r *ExpanderReconciler) enqueueAllFromGVK(ctx context.Context, _ client.Obj
 	return reqs
 }
 
+func (r *ExpanderReconciler) reconcileDelete(ctx context.Context, logger logr.Logger, inputcr unstructured.Unstructured) (ctrl.Result, error) {
+	logger = logger.WithName("Delete")
+	if !controllerutil.ContainsFinalizer(&inputcr, finalizerName) {
+		return ctrl.Result{}, nil
+	}
+	planNN := types.NamespacedName{
+		Name:      r.InputGVR.Resource + "-" + inputcr.GetName(),
+		Namespace: inputcr.GetNamespace(),
+	}
+	plancr := compositionv1alpha1.Plan{}
+	if err := r.Client.Get(ctx, planNN, &plancr); err != nil {
+		logger.Error(err, "Unable to fetch Plan Object", "plan", planNN)
+		return ctrl.Result{}, err
+	}
+	annotations := plancr.GetAnnotations()
+	planLabels := plancr.GetLabels()
+	stageList, ok := annotations[stagesAnnotation]
+	if !ok {
+		err := fmt.Errorf("Plan is missing stage order annotation")
+		logger.Error(err, "Unable to fetch stage order from Plan", "Plan", planNN)
+		return ctrl.Result{}, err
+	}
+	stages := strings.Split(stageList, ",")
+	numFound := 0
+	for i := len(stages) - 1; i >= 0; i-- {
+		r.Recorder.Eventf(&inputcr, corev1.EventTypeNormal, "Delete", "Deleting objects for stage %s", stages[i])
+		nsList, ok := annotations[nsAnnotation]
+		if !ok {
+			err := fmt.Errorf("Plan is missing Namespace annotation")
+			logger.Error(err, "Unable to fetch Namespaces from Plan", "Plan", planNN)
+			return ctrl.Result{}, err
+		}
+		namespaces := strings.Split(nsList, ",")
+		namespaces = append(namespaces, inputcr.GetNamespace())
+		opts, err := deleteListOpts(stages[i], planLabels[applysetIdLabel])
+		if err != nil {
+			logger.Error(err, "Error creating list options")
+			return ctrl.Result{}, err
+		}
+		gkList, ok := annotations[gkAnnotation]
+		if !ok {
+			err := fmt.Errorf("Plan is missing GroupKind annotation")
+			logger.Error(err, "Unable to fetch GroupKinds from Plan", "Plan", planNN)
+			return ctrl.Result{}, err
+		}
+		gks := strings.Split(gkList, ",")
+		for _, gk := range gks {
+			parsedGK := schema.ParseGroupKind(gk)
+			mapping, err := r.RESTMapper.RESTMapping(parsedGK)
+			if err != nil {
+				return ctrl.Result{}, err
+			}
+			n := 0
+			if mapping.Scope.Name() == meta.RESTScopeNameNamespace {
+				n, err = r.deleteNamespacedResources(ctx, logger, stages[i], mapping.Resource, namespaces, opts)
+			} else if mapping.Scope.Name() == meta.RESTScopeNameRoot {
+				n, err = r.deleteClusterResources(ctx, logger, stages[i], mapping.Resource, opts)
+			}
+			if err != nil {
+				logger.Error(err, "Error deleting resources", "GroupKind", gk)
+				r.Recorder.Eventf(&inputcr, corev1.EventTypeWarning, "Delete", "Failed deleting objects of GroupKind %q for stage %q: %v", gk, stages[i], err)
+				return ctrl.Result{}, err
+			}
+			numFound += n
+		}
+		if numFound > 0 {
+			break
+		}
+	}
+	if numFound > 0 {
+		return ctrl.Result{Requeue: true, RequeueAfter: 10 * time.Second}, nil
+	}
+
+	// Remove the finalizers to allow the Plan and Facade to be deleted.
+	controllerutil.RemoveFinalizer(&plancr, finalizerName)
+	if err := r.Update(ctx, &plancr); err != nil {
+		logger.Error(err, "Unable to remove finalizer from Plan")
+		return ctrl.Result{}, err
+	}
+	controllerutil.RemoveFinalizer(&inputcr, finalizerName)
+	if err := r.Update(ctx, &inputcr); err != nil {
+		logger.Error(err, "Unable to remove finalizer from input CR")
+		return ctrl.Result{}, err
+	}
+	// Expanded resources are all deleted, stop reconciliation.
+	return ctrl.Result{}, nil
+}
+
+func deleteListOpts(stage, applysetId string) (metav1.ListOptions, error) {
+	stageReq, err := labels.NewRequirement(applier.StageLabel, selection.Equals, []string{stage})
+	if err != nil {
+		return metav1.ListOptions{}, fmt.Errorf("failed making stage label selector: %v", err)
+	}
+	asReq, err := labels.NewRequirement(partOfLabel, selection.Equals, []string{applysetId})
+	if err != nil {
+		return metav1.ListOptions{}, fmt.Errorf("failed making applysetId label selector: %v", err)
+	}
+	opts := metav1.ListOptions{
+		LabelSelector: labels.NewSelector().Add(*stageReq).Add(*asReq).String(),
+	}
+	return opts, nil
+}
+
+func (r *ExpanderReconciler) deleteNamespacedResources(ctx context.Context, logger logr.Logger, stage string, endpoint schema.GroupVersionResource, namespaces []string, opts metav1.ListOptions) (int, error) {
+	numFound := 0
+	for _, ns := range namespaces {
+		resources, err := r.Dynamic.Resource(endpoint).Namespace(ns).List(ctx, opts)
+		if err != nil {
+			return numFound, fmt.Errorf("error listing resources in Namespace %q for stage %q: %v", ns, stage, err)
+		}
+		for _, res := range resources.Items {
+			logger.Info("Attempting to delete resource", "Resource", res, "Namespace", ns)
+			err := r.Delete(ctx, &res)
+			if err == nil {
+				numFound++
+			} else if err != nil && !apierrors.IsNotFound(err) {
+				return numFound, fmt.Errorf("failed deleting object %v in Namespace %q for stage %q: %v", res, ns, stage, err)
+			}
+		}
+	}
+	return numFound, nil
+}
+
+func (r *ExpanderReconciler) deleteClusterResources(ctx context.Context, logger logr.Logger, stage string, endpoint schema.GroupVersionResource, opts metav1.ListOptions) (int, error) {
+	numFound := 0
+	resources, err := r.Dynamic.Resource(endpoint).List(ctx, opts)
+	if err != nil {
+		return numFound, fmt.Errorf("error listing resources for stage %q: %v", stage, err)
+	}
+	for _, res := range resources.Items {
+		logger.Info("Attempting to delete resource", "Resource", res)
+		err := r.Delete(ctx, &res)
+		if err == nil {
+			numFound++
+		} else if err != nil && !apierrors.IsNotFound(err) {
+			return numFound, fmt.Errorf("failed deleting object %v for stage %q: %v", res, stage, err)
+		}
+	}
+	return numFound, nil
+}
+
 // SetupWithManager sets up the controller with the Manager.
 func (r *ExpanderReconciler) SetupWithManager(mgr ctrl.Manager, cr *unstructured.Unstructured) error {
 	var err error

experiments/compositions/composition/pkg/applier/applier.go

@@ -31,6 +31,8 @@ import (
 	"sigs.k8s.io/kubebuilder-declarative-pattern/pkg/patterns/declarative/pkg/manifest"
 )
 
+const StageLabel = "compositions.google.com/applier-stage"
+
 type ApplierClient struct {
 	RESTMapper meta.RESTMapper
 	Dynamic    *dynamic.DynamicClient
@@ -169,6 +171,7 @@ func (a *Applier) Load() error {
 		a.logger.Error(err, "Error injecting ownerRefs")
 		return err
 	}
+	a.addStageLabel(objects)
 
 	a.objects = []applyset.ApplyableObject{}
 	// loop over objects and extract unstructured
@@ -216,6 +219,13 @@ func (a *Applier) injectOwnerRef(objects *manifest.Objects) error {
 	return nil
 }
 
+func (a *Applier) addStageLabel(objects *manifest.Objects) {
+	labels := map[string]string{StageLabel: a.stageName}
+	for _, o := range objects.Items {
+		o.AddLabels(labels)
+	}
+}
+
 func (a *Applier) getApplyOptions(prune bool) (applyset.Options, error) {
 	var options applyset.Options
 	force := true