From ea9e68b2a598a2baf3d441cf247c3d8c2e8a7d61 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Thu, 9 Nov 2023 12:41:51 -0500 Subject: [PATCH] Wait for server to acknowledge app deletion before trying ignore-cleanup There are certain situations where the app delete succeeds and the call to ignore-cleanup fails because it thinks that the app is not being deleted. This change will retry on this error to avoid the necessity of having to delete the app and then ignore-cleanup. Signed-off-by: Donnie Adams --- pkg/cli/rm_helper.go | 17 ++++++++++++++++- .../apigroups/acorn/apps/ignorecleanup.go | 2 +- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/pkg/cli/rm_helper.go b/pkg/cli/rm_helper.go index 6e6fe95aa..40af58316 100644 --- a/pkg/cli/rm_helper.go +++ b/pkg/cli/rm_helper.go @@ -2,7 +2,9 @@ package cli import ( "context" + "errors" "fmt" + "net/http" "strings" "time" @@ -11,6 +13,7 @@ import ( "github.com/pterm/pterm" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" + k8swait "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/retry" "github.com/acorn-io/runtime/pkg/client" @@ -147,7 +150,19 @@ func removeAcorn(ctx context.Context, c client.Client, arg string, ignoreCleanup } if ignoreCleanup { - if err := c.AppIgnoreDeleteCleanup(ctx, arg); err != nil { + // There are situations where an app being deleted the first time with the --ignore-cleanup flag will fail at this + // step because the server thinks that the app is not being deleted. Retrying here will work around this issue. + if err = retry.OnError(k8swait.Backoff{ + Steps: 5, + Duration: 500 * time.Millisecond, + Factor: 2, + Jitter: 0.1, + }, func(err error) bool { + var statusErr *apierrors.StatusError + return errors.As(err, &statusErr) && statusErr.Status().Code == http.StatusBadRequest && strings.HasSuffix(statusErr.Status().Message, "it is not being deleted") + }, func() error { + return c.AppIgnoreDeleteCleanup(ctx, arg) + }); err != nil { return fmt.Errorf("skipping cleanup for app %s: %w", arg, err) } } diff --git a/pkg/server/registry/apigroups/acorn/apps/ignorecleanup.go b/pkg/server/registry/apigroups/acorn/apps/ignorecleanup.go index b42d3d6d9..5cc736a6f 100644 --- a/pkg/server/registry/apigroups/acorn/apps/ignorecleanup.go +++ b/pkg/server/registry/apigroups/acorn/apps/ignorecleanup.go @@ -59,7 +59,7 @@ func (s *ignoreCleanupStrategy) Create(ctx context.Context, obj types.Object) (t } if app.DeletionTimestamp.IsZero() { - return fmt.Errorf("cannot force delete app %s because it is not being deleted", app.Name) + return apierrors.NewBadRequest(fmt.Sprintf("cannot force delete app %s because it is not being deleted", app.Name)) } // If the app has the destroy job finalizer, remove it to force delete