ROX-15806: Remove pause reconcile annotation before Central deletion (#…

…905)

fleetshard/pkg/central/reconciler/reconciler.go

@@ -39,6 +39,7 @@ const (
 	helmReleaseName = "tenant-resources"
 
 	managedServicesAnnotation = "platform.stackrox.io/managed-services"
+	pauseReconcileAnnotation  = "stackrox.io/pause-reconcile"
 	envAnnotationKey          = "rhacs.redhat.com/environment"
 	clusterNameAnnotationKey  = "rhacs.redhat.com/cluster-name"
 	orgNameAnnotationKey      = "rhacs.redhat.com/org-name"
@@ -722,13 +723,38 @@ func (r *CentralReconciler) ensureCentralCRDeleted(ctx context.Context, central
 
 		return false, errors.Wrapf(err, "delete central CR %s/%s", central.GetNamespace(), central.GetName())
 	}
+
+	// avoid being stuck in a deprovisioning state due to the pause reconcile annotation
+	err = r.disablePauseReconcileIfPresent(ctx, central)
+	if err != nil {
+		return false, err
+	}
+
 	if err := r.client.Delete(ctx, central); err != nil {
 		return false, errors.Wrapf(err, "delete central CR %s/%s", central.GetNamespace(), central.GetName())
 	}
 	glog.Infof("Central CR %s/%s is marked for deletion", central.GetNamespace(), central.GetName())
 	return false, nil
 }
 
+func (r *CentralReconciler) disablePauseReconcileIfPresent(ctx context.Context, central *v1alpha1.Central) error {
+	if central.Annotations == nil {
+		return nil
+	}
+
+	if value, exists := central.Annotations[pauseReconcileAnnotation]; !exists || value != "true" {
+		return nil
+	}
+
+	central.Annotations[pauseReconcileAnnotation] = "false"
+	err := r.client.Update(ctx, central)
+	if err != nil {
+		return fmt.Errorf("removing pause reconcile annotation: %v", err)
+	}
+
+	return nil
+}
+
 func (r *CentralReconciler) ensureChartResourcesExist(ctx context.Context, remoteCentral private.ManagedCentral) error {
 	vals, err := r.chartValues(remoteCentral)
 	if err != nil {

fleetshard/pkg/central/reconciler/reconciler_test.go

@@ -335,6 +335,28 @@ func TestReconcileDelete(t *testing.T) {
 	assert.True(t, k8sErrors.IsNotFound(err))
 }
 
+func TestDisablePauseAnnotation(t *testing.T) {
+	fakeClient := testutils.NewFakeClientBuilder(t).Build()
+	r := NewCentralReconciler(fakeClient, private.ManagedCentral{}, nil, centralDBInitFunc, CentralReconcilerOptions{UseRoutes: true})
+
+	_, err := r.Reconcile(context.TODO(), simpleManagedCentral)
+	require.NoError(t, err)
+
+	central := &v1alpha1.Central{}
+	err = fakeClient.Get(context.TODO(), client.ObjectKey{Name: centralName, Namespace: centralNamespace}, central)
+	require.NoError(t, err)
+	central.Annotations[pauseReconcileAnnotation] = "true"
+	err = fakeClient.Update(context.TODO(), central)
+	require.NoError(t, err)
+
+	err = r.disablePauseReconcileIfPresent(context.TODO(), central)
+	require.NoError(t, err)
+
+	err = fakeClient.Get(context.TODO(), client.ObjectKey{Name: centralName, Namespace: centralNamespace}, central)
+	require.NoError(t, err)
+	require.Equal(t, "false", central.Annotations[pauseReconcileAnnotation])
+}
+
 func TestReconcileDeleteWithManagedDB(t *testing.T) {
 	fakeClient := testutils.NewFakeClientBuilder(t).Build()