From 7d8fc64628fe8cc933017f977509b82e941bcdfa Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Sat, 27 Mar 2021 23:20:23 +0000 Subject: [PATCH] retry kapi.ScaleDeployment on failure --- pkg/kapi/kapi.go | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/pkg/kapi/kapi.go b/pkg/kapi/kapi.go index f3bb322e9c2b..afb699804c2c 100644 --- a/pkg/kapi/kapi.go +++ b/pkg/kapi/kapi.go @@ -212,23 +212,34 @@ func KubectlBinaryPath(version string) string { return path.Join(vmpath.GuestPersistentDir, "binaries", version, "kubectl") } -// ScaleDeployment sets the number of replicas of deployment in namespace and context +// ScaleDeployment tries to set the number of deployment replicas in namespace and context. +// It will retry (usually needed due to "the object has been modified; please apply your changes to the latest version and try again" error) up to ReasonableMutateTime to ensure target scale is achieved. func ScaleDeployment(kcontext, namespace, deploymentName string, replicas int) error { client, err := Client(kcontext) if err != nil { return fmt.Errorf("client: %v", err) } - scale, err := client.AppsV1().Deployments(namespace).GetScale(context.Background(), deploymentName, meta.GetOptions{}) - if err != nil { - return fmt.Errorf("deployment scale: %v", err) - } - if scale.Spec.Replicas != int32(replicas) { - scale.Spec.Replicas = int32(replicas) - _, err = client.AppsV1().Deployments(namespace).UpdateScale(context.Background(), deploymentName, scale, meta.UpdateOptions{}) + err = wait.PollImmediate(kconst.APICallRetryInterval, ReasonableMutateTime, func() (bool, error) { + scale, err := client.AppsV1().Deployments(namespace).GetScale(context.Background(), deploymentName, meta.GetOptions{}) if err != nil { - return fmt.Errorf("deployment rescale: %v", err) + klog.Warningf("failed getting deployment scale, will retry: %v", err) + return false, nil + } + if scale.Spec.Replicas != int32(replicas) { + scale.Spec.Replicas = int32(replicas) + _, err = client.AppsV1().Deployments(namespace).UpdateScale(context.Background(), deploymentName, scale, meta.UpdateOptions{}) + if err != nil { + klog.Warningf("failed rescaling deployment, will retry: %v", err) + } + // repeat (if change was successful - once again to check & confirm requested scale) + return false, nil } + return true, nil + }) + if err != nil { + klog.Infof("timed out trying to rescale deployment %q in namespace %q and context %q to %d: %v", deploymentName, namespace, kcontext, replicas, err) + return err } klog.Infof("deployment %q in namespace %q and context %q rescaled to %d", deploymentName, namespace, kcontext, replicas)